Merge remote-tracking branch 'origin/main' into litellm_prometheus_401_failed_requests_metric

.circleci/config.yml

@@ -2915,6 +2915,25 @@ jobs:
               exit 1
             fi
 
+  provider_replay_harness:
+    docker:
+      - *python312_image
+    working_directory: ~/project
+    resource_class: medium
+    steps:
+      - setup_litellm_test_deps
+      - run:
+          name: Test provider replay harness
+          command: |
+            mkdir -p test-results/provider-replay-harness
+            uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
+              --junitxml=test-results/provider-replay-harness/junit.xml \
+              tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
+              tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
+              tests/code_coverage_tests/test_provider_replay_harness.py
+      - store_test_results:
+          path: test-results/provider-replay-harness
+
   integration_contracts:
     parameters:
       suite:
@@ -2967,6 +2986,7 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - provider_replay_harness
       - base_sdk_install:
           filters: *main_branches
       - local_testing_part1:

.github/e2e-stack/assert_tests_ran.py

@@ -3,6 +3,9 @@ import xml.etree.ElementTree as ET
 from pathlib import Path
 from typing import Final
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
+from coverage_registry.management_cases import MANAGEMENT_CASES
+
 
 def main() -> int:
     selected: Final = tuple(sys.argv[2:])
@@ -16,6 +19,17 @@ def main() -> int:
         case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
     )
     missing: Final = tuple(path for path in selected if path not in passed)
+    required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
+    passed_nodes: Final = frozenset(
+        prop.get("value")
+        for case in cases
+        if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+        for prop in case.findall("./properties/property")
+        if prop.get("name") == "management_node"
+    )
+    missing_nodes: Final = required_nodes - passed_nodes
+    for node in sorted(missing_nodes):
+        _ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
     for path in selected:
         collected: Final = sum(case.get("file") == path for case in cases)
         skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
@@ -27,6 +41,7 @@ def main() -> int:
     if (
         selected
         and not missing
+        and not missing_nodes
         and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
     ):
         return 0

.github/e2e-stack/oidc-profile.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "${REPO_ROOT}"
+exec uv run --no-sync python tests/e2e/idp.py "$@"

.github/e2e-stack/select_tests.py

@@ -12,6 +12,8 @@ UNSUPPORTED: Final = re.compile(
 HARNESS: Final = re.compile(
     r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
     r"|^tests/e2e/idp_realm\.json$"
+    r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$"
+    r"|^tests/e2e/coverage_registry/management_cases\.py$"
     r"|^tests/e2e/gateway/"
     r"|^\.github/e2e-stack/"
     r"|^\.github/workflows/test-e2e-changed\.yml$"

.github/pull_request_template.md

@@ -101,7 +101,8 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
      For bug fixes: Before shows the reproduction, After shows the same steps passing
      For new features: Before shows the capability missing, After shows it working end-to-end
      If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), make each endpoint its own case, not just one
-     For UI changes: before/after screenshots under the same headings -->
+     For UI changes: before/after screenshots under the same headings
+     If the main use case runs through a coding tool like Claude Code or Codex, drive that tool interactively the way the user does (never `claude -p`, `codex exec`, or curl on its own) and embed before/after screenshots of its pane under the same headings; curl replays and headless runs can follow as extra cases, never as the only proof -->
 
 ## Type
 

.github/workflows/_test-unit-base.yml

@@ -57,6 +57,7 @@ permissions:
 
 env:
   UV_PYTHON: "3.12"
+  LITELLM_LOCAL_MODEL_COST_MAP: "True"
 
 jobs:
   run:
@@ -113,6 +114,7 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8
         run: |
+          diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
           uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
 

.github/workflows/test-code-quality.yml

@@ -178,7 +178,7 @@ jobs:
           version: "0.10.9"
 
       - name: Install dependencies
-        run: uv sync --frozen --extra proxy --python 3.10
+        run: uv sync --frozen --extra proxy --extra cli --python 3.10
 
       - run: uv run --no-sync python --version
 
@@ -187,3 +187,6 @@ jobs:
 
       - name: Check litellm CLI
         run: uv run --no-sync litellm --version
+
+      - name: Check lite CLI
+        run: uv run --no-sync lite version

.github/workflows/test-e2e-changed.yml

@@ -183,7 +183,7 @@ jobs:
             log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
             echo "::group::pass ${pass} of 3"
             set +e
-            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \
               -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
             status=$?
             uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"

Makefile

@@ -299,6 +299,9 @@ test-rust-extension:
 	[ "$$#" -eq 1 ] && \
 	UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
 	$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
+	"$$temporary/venv/bin/python" -I -m mypy.stubtest \
+		--mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \
+		litellm.rust_bridge._native && \
 	LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
 	"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust
 

litellm/__init__.py

@@ -538,7 +538,7 @@ context_window_fallbacks: Optional[List] = None
 content_policy_fallbacks: Optional[List] = None
 allowed_fails: int = 3
 allow_dynamic_callback_disabling: bool = True
-num_retries_per_request: Optional[int] = None  # cap on Router retries of one model group; resets per fallback hop
+num_retries_per_request: Optional[int] = None  # for the request overall (incl. fallbacks + model retries)
 ####### SECRET MANAGERS #####################
 secret_manager_client: Optional[Any] = (
     None  # list of instantiated key management clients - e.g. azure kv, infisical, etc.

litellm/anthropic_beta_headers_config.json

@@ -22,6 +22,7 @@
     "mcp-servers-2025-12-04": null,
     "oauth-2025-04-20": "oauth-2025-04-20",
     "output-128k-2025-02-19": "output-128k-2025-02-19",
+    "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
     "skills-2025-10-02": "skills-2025-10-02",
     "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@@ -52,6 +53,7 @@
     "mcp-servers-2025-12-04": null,
     "output-128k-2025-02-19": null,
     "structured-output-2024-03-01": null,
+    "per-turn-control-2026-07-01": null,
     "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
     "skills-2025-10-02": "skills-2025-10-02",
     "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@@ -82,6 +84,7 @@
     "mcp-servers-2025-12-04": null,
     "output-128k-2025-02-19": null,
     "structured-output-2024-03-01": null,
+    "per-turn-control-2026-07-01": null,
     "prompt-caching-scope-2026-01-05": null,
     "skills-2025-10-02": null,
     "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
@@ -113,6 +116,7 @@
     "mcp-servers-2025-12-04": null,
     "output-128k-2025-02-19": null,
     "structured-output-2024-03-01": null,
+    "per-turn-control-2026-07-01": null,
     "prompt-caching-scope-2026-01-05": null,
     "skills-2025-10-02": null,
     "structured-outputs-2025-11-13": null,
@@ -144,6 +148,7 @@
     "mcp-servers-2025-12-04": null,
     "output-128k-2025-02-19": null,
     "structured-output-2024-03-01": null,
+    "per-turn-control-2026-07-01": null,
     "prompt-caching-scope-2026-01-05": null,
     "skills-2025-10-02": null,
     "structured-outputs-2025-11-13": null,
@@ -176,6 +181,7 @@
     "mcp-servers-2025-12-04": null,
     "oauth-2025-04-20": "oauth-2025-04-20",
     "output-128k-2025-02-19": "output-128k-2025-02-19",
+    "per-turn-control-2026-07-01": null,
     "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
     "skills-2025-10-02": "skills-2025-10-02",
     "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",

litellm/batches/batch_utils.py

@@ -9,6 +9,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
 from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
+from litellm.llms.vertex_ai.batches.transformation import vertex_prompt_tokens_details
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import ModelInfo, Usage
 from litellm.utils import token_counter
@@ -356,6 +357,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
             prompt_tokens=_prompt,
             completion_tokens=_completion,
             total_tokens=_total,
+            prompt_tokens_details=vertex_prompt_tokens_details(usage_metadata),
         )
 
         try:

litellm/caching/affinity_cache.py

@@ -0,0 +1,125 @@
+"""Atomic affinity claims shared by deployment and tier-model selection."""
+
+import json
+from collections.abc import Mapping
+from typing import (
+    Final,
+    cast,  # noqa: TID251  # Redis script results are narrowed only to object, then validated
+)
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm._logging import verbose_router_logger
+from litellm.caching.dual_cache import DualCache
+
+_PIN_JSON_ADAPTER: Final = TypeAdapter[JsonValue](JsonValue)
+
+_CLAIM_PIN_SCRIPT: Final = """
+local current = redis.call('GET', KEYS[1])
+if current == false then
+  redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
+  return ARGV[1]
+end
+if ARGV[3] then
+  local decoded, stored = pcall(cjson.decode, current)
+  if decoded and type(stored) == 'table' then
+    for _, eligible in ipairs(cjson.decode(ARGV[3])) do
+      local matches = true
+      for key, value in pairs(eligible) do
+        if stored[key] ~= value then matches = false; break end
+      end
+      for key, _ in pairs(stored) do
+        if eligible[key] == nil then matches = false; break end
+      end
+      if matches then
+        redis.call('EXPIRE', KEYS[1], ARGV[2])
+        return current
+      end
+    end
+  end
+  redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
+  return ARGV[1]
+end
+if current == ARGV[1] then
+  redis.call('EXPIRE', KEYS[1], ARGV[2])
+end
+return current
+"""
+
+
+def set_local_affinity_pin(cache: DualCache, cache_key: str, value: object, ttl_seconds: int) -> None:
+    """Replace the entry because InMemoryCache.set_cache preserves a live key's expiry."""
+    cache.in_memory_cache.delete_cache(cache_key)
+    cache.in_memory_cache.set_cache(cache_key, value, ttl=ttl_seconds)
+
+
+def _legacy_pin_matches(stored: object, pin_value: Mapping[str, str]) -> bool:
+    if isinstance(stored, dict):
+        return all(stored.get(key) is not None and str(stored[key]) == value for key, value in pin_value.items())
+    return isinstance(stored, str) and len(pin_value) == 1 and stored in pin_value.values()
+
+
+def claim_affinity_pin_in_memory(
+    cache: DualCache,
+    cache_key: str,
+    pin_value: Mapping[str, str],
+    ttl_seconds: int,
+    *,
+    eligible_values: tuple[Mapping[str, str], ...] | None = None,
+) -> object:
+    """No await between read and write, so same-loop claims agree during a Redis outage."""
+    existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
+    if existing is not None and eligible_values is None:
+        if _legacy_pin_matches(existing, pin_value):
+            set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
+        return existing
+    winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
+    set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
+    return winner
+
+
+def _decode_pin(value: str) -> object:
+    try:
+        return _PIN_JSON_ADAPTER.validate_json(value)
+    except ValidationError:
+        return value
+
+
+async def claim_affinity_pin(
+    cache: DualCache,
+    cache_key: str,
+    pin_value: Mapping[str, str],
+    ttl_seconds: int,
+    *,
+    eligible_values: tuple[Mapping[str, str], ...] | None = None,
+) -> object:
+    """Return the authoritative first writer, replacing it only when it becomes ineligible.
+
+    Eligible claims refresh the returned winner. Legacy deployment claims only refresh
+    a matching candidate. Resolve Redis per call because the proxy attaches it lazily.
+    """
+    redis_cache: Final = cache.redis_cache
+    if redis_cache is not None:
+        try:
+            claim_script: Final = redis_cache.async_register_script(_CLAIM_PIN_SCRIPT)
+            args: Final = (
+                json.dumps(dict(pin_value)),  # mutable-ok: JSON serialization requires dict, not a generic Mapping
+                int(ttl_seconds),
+                *(
+                    (json.dumps(tuple(dict(value) for value in eligible_values)),)  # mutable-ok: JSON requires dict
+                    if eligible_values is not None
+                    else ()
+                ),
+            )
+            raw: Final = cast(  # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
+                object, await claim_script(keys=(cache_key,), args=args)
+            )
+            decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw
+            if not isinstance(decoded, str):
+                return pin_value
+            winner: Final = _decode_pin(decoded)
+            set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
+            return winner
+        except Exception as error:  # noqa: BLE001  # Redis/Lua faults retain same-pod affinity through local claims
+            verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
+    return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)

litellm/compression/compress.py

@@ -205,21 +205,35 @@ def _extract_anthropic_tool_exchange_spans(
     return spans, None
 
 
+def _message_has_cache_control(message: Mapping[str, object]) -> bool:
+    if message.get("cache_control") is not None:
+        return True
+    content: Final = message.get("content")
+    if isinstance(content, list):
+        return any(isinstance(part, Mapping) and part.get("cache_control") is not None for part in content)
+    return False
+
+
 def get_protected_indices(messages: Sequence[Mapping[str, object]]) -> tuple[int, ...]:
     """
     Return indices of messages that must never be compressed:
     - All system messages
     - The last user message
     - The last assistant message
+    - Any message carrying an Anthropic cache_control breakpoint
 
     The last user message is what the model is being asked to act on right now,
     so compressing it replaces the live instruction with a marker. Compression
-    guardrails share this policy; see the Headroom guardrail.
+    guardrails share this policy; see the Headroom guardrail. A cache_control
+    breakpoint pins the provider's prompt-cache prefix to that row's exact
+    bytes, so rewriting a marked row anywhere in history turns the next
+    request's cache read into a cache write.
     """
     system_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "system")
     last_user: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "user")[-1:]
-    last_assistant = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")[-1:]
-    return system_indices + last_user + last_assistant
+    assistant_indices: Final = tuple(index for index, msg in enumerate(messages) if msg.get("role", "") == "assistant")
+    cache_control_indices: Final = tuple(index for index, msg in enumerate(messages) if _message_has_cache_control(msg))
+    return tuple(dict.fromkeys(system_indices + last_user + assistant_indices[-1:] + cache_control_indices))
 
 
 def _combine_scores(
@@ -421,7 +435,7 @@ def compress(
         combined_scores = bm25_scores
 
     # Protected messages are never compressed
-    protected_indices: Final = get_protected_indices(normalized_messages)
+    protected_indices: Final = get_protected_indices(original_messages)
     kept_indices: set[int] = set(protected_indices)
 
     tool_exchange_spans: list[set[int]] = []

litellm/constants.py

@@ -1976,6 +1976,8 @@ BROWSER_SECURITY_HEADERS: Final[frozenset[str]] = frozenset(
 
 UNSAFE_PROXY_RESPONSE_HEADERS: Final[frozenset[str]] = HTTP_FRAMING_HEADERS | BROWSER_SECURITY_HEADERS
 
+STRINGIFIED_NONE: Final[str] = "None"
+
 # A retrieved response replays the usage of the call that created it, so pricing these
 # read/management routes like inference bills the same tokens twice.
 NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset(

litellm/cost_calculator.py

@@ -2278,6 +2278,19 @@ def default_video_cost_calculator(
     return 0.0
 
 
+def _batch_rate(
+    model_info: ModelInfo,
+    key: Literal[
+        "input_cost_per_audio_token_batches",
+        "input_cost_per_image_token_batches",
+        "input_cost_per_video_token_batches",
+    ],
+    fallback: float,
+) -> float:
+    rate: Final = model_info.get(key)
+    return fallback if rate is None else rate
+
+
 def batch_cost_calculator(
     usage: Usage,
     model: str,
@@ -2337,7 +2350,29 @@ def batch_cost_calculator(
     total_prompt_cost = 0.0
     total_completion_cost = 0.0
     if input_cost_per_token_batches is not None:
-        total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
+        batch_details: Final = parse_prompt_tokens_details(usage)
+        audio_tokens, image_tokens, video_tokens = (
+            batch_details["audio_tokens"],
+            batch_details["image_tokens"],
+            batch_details["video_tokens"],
+        )
+        modality_rates: Final = (
+            _batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches),
+            _batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches),
+            _batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches),
+        )
+        total_prompt_cost = sum(
+            tokens * rate
+            for tokens, rate in zip(
+                (
+                    max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0),
+                    audio_tokens,
+                    image_tokens,
+                    video_tokens,
+                ),
+                (input_cost_per_token_batches, *modality_rates),
+            )
+        )
     elif input_cost_per_token:
         details: Final = parse_prompt_tokens_details(usage)
         cache_read_tokens: Final = details["cache_hit_tokens"]

litellm/integrations/custom_logger.py

@@ -822,46 +822,33 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
     def truncate_standard_logging_payload_content(
         self,
         standard_logging_object: StandardLoggingPayload,
-    ):
+    ) -> StandardLoggingPayload:
         """
-        Truncate error strings and message content in logging payload
+        Return a copy of the logging payload with error_str, messages, and response truncated
 
         Some loggers like DataDog/ GCS Bucket have a limit on the size of the payload. (1MB)
 
-        This function truncates the error string and the message content if they exceed a certain length.
+        Every callback of a request shares one standard logging object, so the payload passed in is left
+        untouched and the callbacks that run later (the prompt caching router check, spend logs) still see
+        the original fields.
         """
-        MAX_STR_LENGTH: Final = 10_000
+        max_str_length: Final = 10_000
+        candidates: Final = {
+            field: self._truncate_field(field_value=standard_logging_object.get(field), max_length=max_str_length)
+            for field in ("error_str", "messages", "response")
+        }
+        truncated_fields: Final = {field: text for field, text in candidates.items() if text is not None}
+        return {**standard_logging_object, **truncated_fields}
 
-        # Truncate fields that might exceed max length
-        fields_to_truncate: Final = ["error_str", "messages", "response"]
-        for field in fields_to_truncate:
-            self._truncate_field(
-                standard_logging_object=standard_logging_object,
-                field_name=field,
-                max_length=MAX_STR_LENGTH,
-            )
-
-    def _truncate_field(
-        self,
-        standard_logging_object: StandardLoggingPayload,
-        field_name: str,
-        max_length: int,
-    ) -> None:
+    def _truncate_field(self, field_value: object, max_length: int) -> str | None:
         """
-        Helper function to truncate a field in the logging payload
+        Return the truncated text of a field that exceeds max_length, or None when the field fits
 
-        This converts the field to a string and then truncates it if it exceeds the max length.
-
-        Why convert to string ?
-        1. User was sending a poorly formatted list for `messages` field, we could not predict where they would send content
-            - Converting to string and then truncating the logged content catches this
-        2. We want to avoid modifying the original `messages`, `response`, and `error_str` in the logging payload since these are in kwargs and could be returned to the user
+        The field is measured as a string because users send poorly formatted lists for `messages`, so there is
+        no fixed place the content would be.
         """
-        field_value: Final[object] = standard_logging_object.get(field_name)
-        if field_value:
-            str_value: Final = str(field_value)
-            if len(str_value) > max_length:
-                standard_logging_object[field_name] = self._truncate_text(text=str_value, max_length=max_length)
+        text: Final = str(field_value or "")
+        return self._truncate_text(text=text, max_length=max_length) if len(text) > max_length else None
 
     def _truncate_text(self, text: str, max_length: int) -> str:
         """Truncate text if it exceeds max_length"""

litellm/integrations/datadog/datadog.py

@@ -563,11 +563,10 @@ class DataDogLogger(
         if standard_logging_object.get("status") == "failure":
             status = DataDogStatus.ERROR
 
-        # Build the initial payload
-        self.truncate_standard_logging_payload_content(standard_logging_object)
+        truncated_payload: Final = self.truncate_standard_logging_payload_content(standard_logging_object)
 
         dd_payload: Final = self._create_datadog_logging_payload_helper(
-            standard_logging_object=standard_logging_object,
+            standard_logging_object=truncated_payload,
             status=status,
         )
         return dd_payload

litellm/litellm_core_utils/core_helpers.py

@@ -309,8 +309,8 @@ def max_retries_per_request_hit(kwargs: Mapping[str, object], num_retries_per_re
     metadata: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs))
     if not isinstance(metadata, Mapping):
         return False
-    attempted_retries: Final = metadata.get("attempted_retries")
-    return type(attempted_retries) is int and 0 < attempted_retries and num_retries_per_request <= attempted_retries
+    retry_count: Final = metadata.get("request_retry_count")
+    return type(retry_count) is int and 0 < retry_count and num_retries_per_request <= retry_count
 
 
 def get_or_create_metadata_bucket(

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -1,6 +1,9 @@
+import inspect
 import json
 import re
 import traceback
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Any, Final, Protocol, cast
 
 import httpx
@@ -202,11 +205,17 @@ def _get_response_headers(original_exception: Exception) -> httpx.Headers | None
     return _response_headers
 
 
+def _accepted_init_kwargs(exception_class: type[Exception], candidates: Mapping[str, object]) -> Mapping[str, object]:
+    accepted: Final = inspect.signature(exception_class).parameters
+    return MappingProxyType({name: value for name, value in candidates.items() if name in accepted})
+
+
 def extract_and_raise_litellm_exception(
     response: Any | None,
     error_str: str,
     model: str,
     custom_llm_provider: str,
+    body: object | None = None,
 ):
     """
     Covers scenario where litellm sdk calling proxy.
@@ -216,32 +225,19 @@ def extract_and_raise_litellm_exception(
     Relevant Issue: https://github.com/BerriAI/litellm/issues/7259
     """
     pattern: Final = r"litellm\.\w+Error"
-
-    # Search for the exception in the error string
     match: Final = re.search(pattern, error_str)
-
-    # Extract the exception if found
-    if match:
-        exception_name = match.group(0)
-        exception_name = exception_name.strip().replace("litellm.", "")
-        raised_exception_obj: Final = getattr(litellm, exception_name, None)
-        if raised_exception_obj:
-            # Try with response parameter first, fall back to without it
-            # Some exceptions (e.g., APIConnectionError) don't accept response param
-            try:
-                raise raised_exception_obj(
-                    message=error_str,
-                    llm_provider=custom_llm_provider,
-                    model=model,
-                    response=response,
-                )
-            except TypeError:
-                # Exception doesn't accept response parameter
-                raise raised_exception_obj(
-                    message=error_str,
-                    llm_provider=custom_llm_provider,
-                    model=model,
-                )
+    if match is None:
+        return
+    exception_name: Final = match.group(0).removeprefix("litellm.")
+    raised_exception_obj: Final = getattr(litellm, exception_name, None)
+    if not raised_exception_obj:
+        return
+    raise raised_exception_obj(
+        message=error_str,
+        llm_provider=custom_llm_provider,
+        model=model,
+        **_accepted_init_kwargs(raised_exception_obj, MappingProxyType({"response": response, "body": body})),
+    )
 
 
 class _ProviderHTTPException(Protocol):
@@ -254,6 +250,23 @@ class _ProviderHTTPException(Protocol):
     llm_provider: str
 
 
+def _litellm_proxy_response(
+    original_exception: _ProviderHTTPException, custom_llm_provider: str
+) -> httpx.Response | None:
+    response: Final = getattr(original_exception, "response", None)
+    if custom_llm_provider != "litellm_proxy" or not isinstance(response, httpx.Response) or response.headers:
+        return response
+    headers: Final = getattr(original_exception, "headers", None)
+    if not isinstance(headers, Mapping) or not headers:
+        return response
+    pairs: Final = headers.multi_items() if isinstance(headers, httpx.Headers) else headers.items()
+    return httpx.Response(
+        status_code=response.status_code,
+        headers=[(str(k), str(v)) for k, v in pairs],
+        request=getattr(original_exception, "request", None),
+    )
+
+
 def _map_openai_exception(
     *,
     model: str,
@@ -264,6 +277,7 @@ def _map_openai_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
+    response: Final = _litellm_proxy_response(original_exception, custom_llm_provider)
     # custom_llm_provider is openai, make it OpenAI
     message = get_error_message(error_obj=original_exception)
     if message is None:
@@ -292,14 +306,14 @@ def _map_openai_exception(
             message=f"RateLimitError: {exception_provider} - {message}",
             model=model,
             llm_provider=custom_llm_provider,
-            response=getattr(original_exception, "response", None),
+            response=response,
         )
     elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
         raise ContextWindowExceededError(
             message=f"ContextWindowExceededError: {exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif "invalid_request_error" in error_str and "model_not_found" in error_str:
@@ -307,7 +321,7 @@ def _map_openai_exception(
             message=f"{exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif "A timeout occurred" in error_str:
@@ -326,8 +340,9 @@ def _map_openai_exception(
             message=f"ContentPolicyViolationError: {exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
+            body=getattr(original_exception, "body", None),
         )
     elif "invalid_encrypted_content" in error_str or "could not be verified" in error_str:
         helpful_message: Final = (
@@ -345,7 +360,7 @@ def _map_openai_exception(
             message=helpful_message,
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
             body=getattr(original_exception, "body", None),
         )
@@ -354,7 +369,7 @@ def _map_openai_exception(
             message=f"{exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
             body=getattr(original_exception, "body", None),
         )
@@ -372,7 +387,7 @@ def _map_openai_exception(
             message=f"RateLimitError: {exception_provider} - {message}",
             model=model,
             llm_provider=custom_llm_provider,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif (
@@ -383,7 +398,7 @@ def _map_openai_exception(
             message=f"AuthenticationError: {exception_provider} - {message}",
             llm_provider=custom_llm_provider,
             model=model,
-            response=getattr(original_exception, "response", None),
+            response=response,
             litellm_debug_info=extra_information,
         )
     elif "Mistral API raised a streaming error" in error_str:
@@ -402,15 +417,16 @@ def _map_openai_exception(
                 message=f"{exception_provider} - {message}",
                 llm_provider=custom_llm_provider,
                 model=model,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
+                body=getattr(original_exception, "body", None),
             )
         elif original_exception.status_code == 401:
             raise AuthenticationError(
                 message=f"AuthenticationError: {exception_provider} - {message}",
                 llm_provider=custom_llm_provider,
                 model=model,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 404:
@@ -418,7 +434,7 @@ def _map_openai_exception(
                 message=f"NotFoundError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 408:
@@ -433,7 +449,7 @@ def _map_openai_exception(
                 message=f"{exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
                 body=getattr(original_exception, "body", None),
             )
@@ -442,7 +458,7 @@ def _map_openai_exception(
                 message=f"RateLimitError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 500:
@@ -450,7 +466,7 @@ def _map_openai_exception(
                 message=f"InternalServerError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 502:
@@ -458,7 +474,7 @@ def _map_openai_exception(
                 message=f"BadGatewayError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 503:
@@ -466,7 +482,7 @@ def _map_openai_exception(
                 message=f"ServiceUnavailableError: {exception_provider} - {message}",
                 model=model,
                 llm_provider=custom_llm_provider,
-                response=getattr(original_exception, "response", None),
+                response=response,
                 litellm_debug_info=extra_information,
             )
         elif original_exception.status_code == 504:  # gateway timeout error
@@ -2423,10 +2439,11 @@ def exception_type(
                 custom_llm_provider == "litellm_proxy"
             ):  # handle special case where calling litellm proxy + exception str contains error message
                 extract_and_raise_litellm_exception(
-                    response=getattr(original_exception, "response", None),
+                    response=_litellm_proxy_response(mappable_exception, custom_llm_provider),
                     error_str=error_str,
                     model=model,
                     custom_llm_provider=custom_llm_provider,
+                    body=getattr(original_exception, "body", None),
                 )
             if (
                 custom_llm_provider == "openai"

litellm/litellm_core_utils/litellm_logging.py

@@ -644,6 +644,24 @@ class Logging(LiteLLMLoggingBaseClass):
         """Keep ``_response_ms`` / ``litellm_overhead_time_ms`` for a result that has no ``_hidden_params``."""
         self.response_timing_metrics = dict(timing_metrics)  # mutable-ok: kept deep-copyable
 
+    def add_dynamic_callback(self, callback: CustomLogger) -> None:
+        self.dynamic_input_callbacks = self._with_dynamic_callback(self.dynamic_input_callbacks, callback)
+        self.dynamic_success_callbacks = self._with_dynamic_callback(self.dynamic_success_callbacks, callback)
+        self.dynamic_async_success_callbacks = self._with_dynamic_callback(
+            self.dynamic_async_success_callbacks, callback
+        )
+        self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, callback)
+        self.dynamic_async_failure_callbacks = self._with_dynamic_callback(
+            self.dynamic_async_failure_callbacks, callback
+        )
+
+    @staticmethod
+    def _with_dynamic_callback(
+        callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger
+    ) -> list[str | Callable | CustomLogger]:
+        existing: Final = tuple(callbacks or ())
+        return [*existing, *(() if callback in existing else (callback,))]
+
     def process_dynamic_callbacks(self):
         """
         Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -484,11 +484,12 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
 def _apply_off_peak_to_base_costs(
     model_info: ModelInfo,
     current_time: datetime | None,
-    base_costs: tuple[float, float, float, float, float],
+    base_costs: tuple[float, float, float, float | None, float],
 ) -> tuple[float, float, float, float, float]:
     """Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
-    produced them. The one-hour cache-creation rate passes through untouched, since
-    off_peak_pricing has no field for it, and reasoning is left to _resolve_billed_reasoning_rate.
+    produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
+    present one passes through untouched and an absent one resolves to the applied
+    cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
     """
     prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
     rates: Final = apply_off_peak_pricing(
@@ -506,7 +507,7 @@ def _apply_off_peak_to_base_costs(
         rates.input_rate,
         rates.output_rate,
         rates.cache_creation_rate,
-        cache_creation_above_1hr,
+        rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
         rates.cache_read_rate,
     )
 
@@ -532,6 +533,11 @@ def _get_token_base_cost(
     `missing_cache_read_uses_input` resolves an absent cache-read rate to the resolved
     input rate instead of 0.0; an explicit 0.0 rate stays a real price either way.
 
+    An absent cache-creation rate always resolves to the resolved input rate, the way the
+    tiered table and custom deployment pricing already do, since a provider that publishes
+    no write price bills cache writes as ordinary input. An absent 1h write rate resolves
+    to the cache-creation rate, off-peak included. An explicit 0.0 stays a real price for both.
+
     Returns:
         Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
     """
@@ -554,10 +560,9 @@ def _get_token_base_cost(
         output_image_cost: Final = _get_cost_per_unit(model_info, "output_cost_per_image_token", None)
         if output_image_cost is not None:
             completion_base_cost = cast(float, output_image_cost)
-    cache_creation_cost = cast(float, _get_cost_per_unit(model_info, cache_creation_cost_key))
-    cache_creation_cost_above_1hr = cast(
-        float,
-        _get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
+    cache_creation_cost = _get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
+    cache_creation_cost_above_1hr = _get_cost_per_unit(
+        model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
     )
     cache_read_cost = _get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
 
@@ -639,22 +644,10 @@ def _get_token_base_cost(
                         else f"cache_read_input_token_cost_above_{threshold_str}_tokens"
                     )
 
-                    cache_creation_cost = cast(
-                        float,
-                        _get_cost_per_unit(
-                            model_info,
-                            cache_creation_tiered_key,
-                            cache_creation_cost,
-                        ),
-                    )
+                    cache_creation_cost = _get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
 
-                    cache_creation_cost_above_1hr = cast(
-                        float,
-                        _get_cost_per_unit(
-                            model_info,
-                            cache_creation_1hr_tiered_key,
-                            cache_creation_cost_above_1hr,
-                        ),
+                    cache_creation_cost_above_1hr = _get_cost_per_unit(
+                        model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
                     )
 
                     cache_read_cost = _get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
@@ -665,16 +658,16 @@ def _get_token_base_cost(
             except Exception:
                 continue
 
+    input_rate_for_missing_cache_rates: Final = _off_peak_rate(
+        _open_off_peak_block(model_info, current_time) or MappingProxyType({}),
+        "input_cost_per_token",
+        prompt_base_cost,
+    )
     if cache_read_cost is None:
-        cache_read_cost = (
-            _off_peak_rate(
-                _open_off_peak_block(model_info, current_time) or MappingProxyType({}),
-                "input_cost_per_token",
-                prompt_base_cost,
-            )
-            if missing_cache_read_uses_input
-            else 0.0
-        )
+        cache_read_cost = input_rate_for_missing_cache_rates if missing_cache_read_uses_input else 0.0
+    resolved_cache_creation_cost: Final = (
+        input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
+    )
 
     return _apply_off_peak_to_base_costs(
         model_info,
@@ -682,7 +675,7 @@ def _get_token_base_cost(
         (
             prompt_base_cost,
             completion_base_cost,
-            cache_creation_cost,
+            resolved_cache_creation_cost,
             cache_creation_cost_above_1hr,
             cache_read_cost,
         ),
@@ -956,12 +949,16 @@ def _calculate_input_cost(
     )
 
     ### AUDIO COST
-    if prompt_tokens_details["audio_tokens"]:
+    if prompt_tokens_details["audio_tokens"] and not (
+        prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
+    ):
         audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
         prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
 
     ### IMAGE TOKEN COST
-    if prompt_tokens_details["image_tokens"]:
+    if prompt_tokens_details["image_tokens"] and not (
+        prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
+    ):
         # For image token costs:
         # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
         image_token_cost_key = "input_cost_per_image_token"
@@ -970,7 +967,9 @@ def _calculate_input_cost(
         prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
 
     ### VIDEO TOKEN COST
-    if prompt_tokens_details["video_tokens"]:
+    if prompt_tokens_details["video_tokens"] and not (
+        prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
+    ):
         video_token_cost_key = "input_cost_per_video_token"
         if model_info.get(video_token_cost_key) is None:
             video_token_cost_key = "input_cost_per_token"

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -20,6 +20,7 @@ from itertools import chain, repeat
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
 
+from pydantic import TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict, assert_never
 
 from litellm._logging import verbose_proxy_logger
@@ -44,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     scoped_structured_message_indices,
     stream_item_field,
     stream_item_fingerprint,
+    unappliable_request_rewrite,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
     AnthropicPassthroughLoggingHandler,
@@ -103,9 +105,24 @@ class ToolResultBlockTextTarget:
     block_idx: int
 
 
-InputWriteBackTarget = (
-    MessageContentTarget | ContentBlockTextTarget | ToolResultStringTarget | ToolResultBlockTextTarget
-)
+@dataclass(frozen=True, slots=True)
+class SystemStringTarget:
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class SystemBlockTextTarget:
+    block_idx: int
+
+
+@dataclass(frozen=True, slots=True)
+class ToolUseInputTarget:
+    msg_idx: int
+    content_idx: int
+
+
+MessageTextTarget = MessageContentTarget | ContentBlockTextTarget | ToolResultStringTarget | ToolResultBlockTextTarget
+InputWriteBackTarget = SystemStringTarget | SystemBlockTextTarget | MessageTextTarget
 
 
 def _as_str_mapping(value: Mapping[str, object]) -> Mapping[str, object]:
@@ -146,10 +163,17 @@ class ScannedText:
     target: InputWriteBackTarget
 
 
+@dataclass(frozen=True, slots=True)
+class ScannedToolCall:
+    tool_call: ChatCompletionToolCallChunk
+    target: ToolUseInputTarget
+
+
 @dataclass(frozen=True, slots=True)
 class ExtractedInput:
     scanned: tuple[ScannedText, ...]
     images: tuple[str, ...]
+    tool_calls: tuple[ScannedToolCall, ...] = ()
 
 
 EMPTY_EXTRACTED_INPUT: Final = ExtractedInput(scanned=(), images=())
@@ -161,6 +185,74 @@ class _ToolCallShape:
     arguments: str
 
 
+def _is_client_tool_use(block: Mapping[str, object]) -> bool:
+    return (
+        block.get("type") == "tool_use"
+        and isinstance(block.get("id"), str)
+        and isinstance(block.get("name"), str)
+        and isinstance(block.get("input"), dict)
+    )
+
+
+def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
+    if not isinstance(system, list):
+        return
+    text_blocks: Final = tuple(block for block in system if isinstance(block, dict) and block.get("type") == "text")
+    if block_idx < len(text_blocks):
+        text_blocks[block_idx]["text"] = (
+            response  # mutable-ok: guardrails rewrite the caller's request payload in place
+        )
+
+
+def _write_back_message_text(message: _WritableMessage, target: MessageTextTarget, response: str) -> None:
+    content: Final = message.get("content", None)
+    if content is None:
+        return
+    match target:
+        case MessageContentTarget():
+            if isinstance(content, str):
+                message["content"] = response  # mutable-ok: guardrails rewrite the caller's request payload in place
+        case ContentBlockTextTarget(content_idx=content_idx):
+            if isinstance(content, list):
+                content[content_idx]["text"] = (
+                    response  # mutable-ok: guardrails rewrite the caller's request payload in place
+                )
+        case ToolResultStringTarget(content_idx=content_idx):
+            if isinstance(content, list):
+                content[content_idx]["content"] = (
+                    response  # mutable-ok: guardrails rewrite the caller's request payload in place
+                )
+        case ToolResultBlockTextTarget(content_idx=content_idx, block_idx=block_idx):
+            if isinstance(content, list):
+                content[content_idx]["content"][block_idx]["text"] = (
+                    response  # mutable-ok: guardrails rewrite the caller's request payload in place
+                )
+        case _:
+            assert_never(target)
+
+
+_TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object])
+
+
+def _rewritten_tool_use_input(arguments: str) -> Mapping[str, object] | None:
+    try:
+        return _TOOL_USE_INPUT_ADAPTER.validate_json(arguments)
+    except ValidationError:
+        return None
+
+
+def _write_back_tool_use(
+    message: _WritableMessage, target: ToolUseInputTarget, shape: _ToolCallShape, rewritten_input: Mapping[str, object]
+) -> None:
+    content: Final = message.get("content", None)
+    block: Final = content[target.content_idx] if isinstance(content, list) else None
+    if not isinstance(block, dict):
+        return
+    block["input"] = rewritten_input  # mutable-ok: guardrails rewrite the caller's request payload in place
+    if shape.name is not None and shape.name != block.get("name"):
+        block["name"] = shape.name  # mutable-ok: guardrails rewrite the caller's request payload in place
+
+
 @dataclass(frozen=True, slots=True)
 class _SSEFieldRewrite:
     """One field of one nested section of a buffered SSE event, rewritten."""
@@ -452,9 +544,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
         scan_only_tool_results: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply)
 
-        # Exclude only the trusted top-level prompt. In-sequence system entries are untrusted
-        # and must stay aligned with texts_to_check for positional masking. When the top-level
-        # prompt is included, the pre-existing count mismatch disables positional masking.
+        # The top-level prompt is translated on its own below so it can be hoisted in front of
+        # any mid-turn system entries and scanned first, aligned with that structured position.
         translation_source: Final = {  # mutable-ok: API message payload
             key: value for key, value in data.items() if key != "system"
         }
@@ -490,7 +581,12 @@ class AnthropicMessagesHandler(BaseTranslation):
             ]
         )
 
-        # Step 1: Extract all text content and images
+        # Step 1: Extract all text content, images, and tool calls
+        top_level_system_scanned: Final = (
+            ()
+            if hoisted_system_message is None or scan_only_tool_results
+            else self._extract_top_level_system_text(hoisted_system_message)
+        )
         extracted: Final = tuple(
             self._extract_input_text_and_images(
                 message=message,
@@ -501,17 +597,27 @@ class AnthropicMessagesHandler(BaseTranslation):
             )
             for msg_idx, message in enumerate(messages)
         )
-        scanned: Final = tuple(item for one_message in extracted for item in one_message.scanned)
+        scanned: Final = (
+            *top_level_system_scanned,
+            *(item for one_message in extracted for item in one_message.scanned),
+        )
         texts_to_check: Final = [item.text for item in scanned]  # mutable-ok: GenericGuardrailAPIInputs takes list[str]
         images_to_check: Final = [
             image for one_message in extracted for image in one_message.images
         ]  # mutable-ok: GenericGuardrailAPIInputs takes list[str]
+        scanned_tool_calls: Final = tuple(item for one_message in extracted for item in one_message.tool_calls)
+        tool_calls_to_check: Final = [
+            item.tool_call for item in scanned_tool_calls
+        ]  # mutable-ok: GenericGuardrailAPIInputs takes list[ChatCompletionToolCallChunk]
+        pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_to_check)
 
-        # Step 2: Apply guardrail to all texts in batch
-        if texts_to_check:
+        # Step 2: Apply guardrail to all texts and tool calls in batch
+        if texts_to_check or tool_calls_to_check:
             inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
             if images_to_check:
                 inputs["images"] = images_to_check
+            if tool_calls_to_check:
+                inputs["tool_calls"] = tool_calls_to_check
             if tools_to_check:
                 inputs["tools"] = tools_to_check
             original_structured_messages: Final = structured_messages
@@ -570,9 +676,18 @@ class AnthropicMessagesHandler(BaseTranslation):
                     preserve_system_messages=has_midturn_system_message,
                 )
             else:
+                if guardrailed_texts and len(guardrailed_texts) != len(scanned):
+                    raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
+                self._apply_guardrail_tool_calls_to_input(
+                    messages=messages,
+                    scanned_tool_calls=scanned_tool_calls,
+                    pre_guardrail_tool_calls=pre_guardrail_tool_calls,
+                    returned_tool_calls=guardrailed_inputs.get("tool_calls"),
+                    guardrail_name=guardrail_to_apply.guardrail_name,
+                )
                 # Step 3: Map guardrail responses back to original message structure
                 await self._apply_guardrail_responses_to_input(
-                    messages=messages,
+                    data=data,
                     responses=guardrailed_texts,
                     scanned=scanned,
                 )
@@ -598,6 +713,19 @@ class AnthropicMessagesHandler(BaseTranslation):
         hoisted: Final = probe.get("messages") or []  # mutable-ok: API message payload
         return hoisted[0] if hoisted else None
 
+    @staticmethod
+    def _extract_top_level_system_text(hoisted_system_message: AllMessageValues) -> tuple[ScannedText, ...]:
+        content: Final = hoisted_system_message.get("content")
+        if isinstance(content, str):
+            return (ScannedText(content, SystemStringTarget()),)
+        if not isinstance(content, list):
+            return ()
+        return tuple(
+            ScannedText(text_str, SystemBlockTextTarget(block_idx))
+            for block_idx, block in enumerate(content)
+            if isinstance(block, dict) and isinstance(text_str := block.get("text"), str)
+        )
+
     @staticmethod
     def _openai_system_message_to_anthropic(
         message: Mapping[str, object],
@@ -852,9 +980,25 @@ class AnthropicMessagesHandler(BaseTranslation):
             for content_idx, content_item in enumerate(content)
             if isinstance(content_item, dict)
         )
+        tool_use_blocks: Final = (
+            ()
+            if scan_only_tool_results
+            else tuple(
+                (content_idx, content_item)
+                for content_idx, content_item in enumerate(content)
+                if isinstance(content_item, dict) and _is_client_tool_use(content_item)
+            )
+        )
         return ExtractedInput(
             scanned=tuple(item for block in blocks for item in block.scanned),
             images=tuple(image for block in blocks for image in block.images),
+            tool_calls=tuple(
+                ScannedToolCall(
+                    tool_call=AnthropicConfig.convert_tool_use_to_openai_format(content_item, tool_call_idx),
+                    target=ToolUseInputTarget(msg_idx, content_idx),
+                )
+                for tool_call_idx, (content_idx, content_item) in enumerate(tool_use_blocks)
+            ),
         )
 
     @classmethod
@@ -940,43 +1084,59 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     async def _apply_guardrail_responses_to_input(
         self,
-        messages: Sequence[_WritableMessage],
-        responses: list[str],
+        data: dict[str, object],  # mutable-ok: API message payload
+        responses: Sequence[str],
         scanned: tuple[ScannedText, ...],
     ) -> None:
         """
-        Apply guardrail responses back to input messages.
+        Apply guardrail responses back to the top-level system prompt and the input messages.
         """
+        raw_messages: Final = data.get("messages")
+        messages: Final[Sequence[_WritableMessage]] = raw_messages if isinstance(raw_messages, list) else ()
         for item, guardrail_response in zip(scanned, responses):
-            target = item.target
-            message = messages[target.msg_idx]
-            content = message.get("content", None)
-            if content is None:
-                continue
-
-            match target:
-                case MessageContentTarget():
-                    if isinstance(content, str):
-                        message["content"] = (
-                            guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
-                        )
-                case ContentBlockTextTarget(content_idx=content_idx):
-                    if isinstance(content, list):
-                        content[content_idx]["text"] = (
-                            guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
-                        )
-                case ToolResultStringTarget(content_idx=content_idx):
-                    if isinstance(content, list):
-                        content[content_idx]["content"] = (
-                            guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
-                        )
-                case ToolResultBlockTextTarget(content_idx=content_idx, block_idx=block_idx):
-                    if isinstance(content, list):
-                        content[content_idx]["content"][block_idx]["text"] = (
+            match item.target:
+                case SystemStringTarget():
+                    if isinstance(data.get("system"), str):
+                        data["system"] = (
                             guardrail_response  # mutable-ok: guardrails rewrite the caller's request payload in place
                         )
+                case SystemBlockTextTarget(block_idx=block_idx):
+                    _write_back_system_block(data.get("system"), block_idx, guardrail_response)
+                case (
+                    MessageContentTarget()
+                    | ContentBlockTextTarget()
+                    | ToolResultStringTarget()
+                    | ToolResultBlockTextTarget() as message_target
+                ):
+                    _write_back_message_text(messages[message_target.msg_idx], message_target, guardrail_response)
                 case _:
-                    assert_never(target)
+                    assert_never(item.target)
+
+    @staticmethod
+    def _apply_guardrail_tool_calls_to_input(
+        messages: Sequence[_WritableMessage],
+        scanned_tool_calls: tuple[ScannedToolCall, ...],
+        pre_guardrail_tool_calls: tuple[_ToolCallShape, ...],
+        returned_tool_calls: Sequence[object] | None,
+        guardrail_name: str | None,
+    ) -> None:
+        post_guardrail_tool_calls: Final = _tool_call_shapes(
+            returned_tool_calls
+            if returned_tool_calls is not None and len(returned_tool_calls) == len(pre_guardrail_tool_calls)
+            else tuple(item.tool_call for item in scanned_tool_calls)
+        )
+        rewritten: Final = tuple(
+            (item, after, _rewritten_tool_use_input(after.arguments))
+            for item, before, after in zip(scanned_tool_calls, pre_guardrail_tool_calls, post_guardrail_tool_calls)
+            if before != after
+        )
+        applicable: Final = tuple(
+            (item, after, rewritten_input) for item, after, rewritten_input in rewritten if rewritten_input is not None
+        )
+        if len(applicable) != len(rewritten):
+            raise unappliable_request_rewrite(guardrail_name)
+        for item, after, rewritten_input in applicable:
+            _write_back_tool_use(messages[item.target.msg_idx], item.target, after, rewritten_input)
 
     async def process_output_response(
         self,

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -42,6 +42,10 @@ DROP_UNFITTING_REASONING_EFFORT_WARNING: Final = (
 )
 
 
+def _messages_carry_output_config(messages: Sequence[object]) -> bool:
+    return any(isinstance(message, Mapping) and "output_config" in message for message in messages)
+
+
 class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     @property
     def custom_llm_provider(self) -> str | None:
@@ -331,6 +335,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         headers = self._update_headers_with_anthropic_beta(
             headers=headers,
             optional_params=optional_params,
+            messages=messages,
         )
 
         return headers, api_base
@@ -664,6 +669,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         headers: dict,
         optional_params: dict,
         custom_llm_provider: str = "anthropic",
+        messages: Sequence[object] = (),
     ) -> dict:
         """
         Auto-inject anthropic-beta headers based on features used.
@@ -673,24 +679,30 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         - tool_search: adds provider-specific tool search header
         - output_format: adds 'structured-outputs-2025-11-13'
         - speed: adds 'fast-mode-2026-02-01'
+        - a message carrying output_config: adds 'per-turn-control-2026-07-01'
 
         Args:
             headers: Request headers dict
             optional_params: Optional parameters including tools, context_management, output_format, speed
             custom_llm_provider: Provider name for looking up correct tool search header
+            messages: Request messages, scanned for per-message output_config
         """
         beta_values: Final[set] = set()
 
-        # Get existing beta headers if any
-        existing_beta: Final = headers.get("anthropic-beta")
-        if existing_beta:
-            beta_values.update(b.strip() for b in existing_beta.split(","))
+        existing_beta: Final = tuple(
+            piece.strip()
+            for key, value in headers.items()
+            if key.lower() == "anthropic-beta"
+            for piece in value.split(",")
+            if piece.strip()
+        )
+        beta_values.update(existing_beta)
 
         # Check for context management
         context_management_param: Final = optional_params.get("context_management")
         if context_management_param is not None:
             # Check edits array for compact_20260112 type
-            edits: Final = context_management_param.get("edits", [])
+            edits: Final = context_management_param.get("edits", ())
             has_compact = False
             has_other = False
 
@@ -722,24 +734,18 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         if optional_params.get("speed") == "fast":
             beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.FAST_MODE_2026_02_01.value)
 
-        # Check for advisor tool
-        tools = optional_params.get("tools")
-        if tools:
-            for tool in tools:
-                if isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE:
-                    beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value)
-                    break
+        if _messages_carry_output_config(messages):
+            beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
 
-        # Check for tool search tools
-        tools = optional_params.get("tools")
-        if tools:
-            anthropic_model_info: Final = AnthropicModelInfo()
-            if anthropic_model_info.is_tool_search_used(tools):
-                # Use provider-specific tool search header
-                tool_search_header: Final = get_tool_search_beta_header(custom_llm_provider)
-                beta_values.add(tool_search_header)
+        tools: Final = optional_params.get("tools")
+        if any(isinstance(tool, dict) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE for tool in tools or ()):
+            beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value)
 
-        if beta_values:
-            headers["anthropic-beta"] = ",".join(sorted(beta_values))
+        if AnthropicModelInfo().is_tool_search_used(tools):
+            beta_values.add(get_tool_search_beta_header(custom_llm_provider))
 
-        return headers
+        if not beta_values:
+            return headers
+        merged: Final = {key: value for key, value in headers.items() if key.lower() != "anthropic-beta"}
+        merged["anthropic-beta"] = ",".join(sorted(beta_values))
+        return merged

litellm/llms/azure_ai/anthropic/messages_transformation.py

@@ -68,6 +68,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
         headers = self._update_headers_with_anthropic_beta(
             headers=headers,
             optional_params=optional_params,
+            messages=messages,
         )
 
         return headers, api_base

litellm/llms/base_llm/guardrail_translation/utils.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import json
-from collections.abc import Callable, Iterator, Sequence
-from typing import Final, TypeVar
+from collections.abc import Callable, Iterator, Mapping, Sequence
+from typing import Final, TypeVar, cast  # noqa: TID251  # a rebuilt chat row has no typed constructor across roles
 
 from pydantic import BaseModel
 
@@ -364,3 +364,67 @@ def merge_guardrailed_scoped_messages(
                 yield from appended
 
     return list(_merged())
+
+
+def _content_part_text(part: object) -> str | None:
+    if not isinstance(part, Mapping):
+        return None
+    text: Final = part.get("text")
+    return text if isinstance(text, str) else None
+
+
+def message_slot_texts(message: Mapping[str, object]) -> tuple[str, ...]:
+    content: Final = message.get("content")
+    if isinstance(content, str):
+        return (content,)
+    if isinstance(content, list):
+        return tuple(text for part in content if (text := _content_part_text(part)) is not None)
+    return ()
+
+
+def message_text_slot_count(message: AllMessageValues) -> int:
+    return len(message_slot_texts(message))
+
+
+def _part_with_text(part: object, text: str) -> object:
+    if not isinstance(part, Mapping):
+        return part
+    return {**part, "text": text}  # mutable-ok: content parts stay JSON-plain dicts
+
+
+def _content_with_slot_texts(content: Sequence[object], texts: Sequence[str]) -> Sequence[object]:
+    remaining_texts: Final = iter(texts)
+    return [  # mutable-ok: message content stays a JSON list
+        _part_with_text(part, next(remaining_texts)) if _content_part_text(part) is not None else part
+        for part in content
+    ]
+
+
+def message_with_slot_texts(message: AllMessageValues, texts: Sequence[str]) -> AllMessageValues | None:
+    """Swap one rewritten text into each text slot of a chat row, in order.
+
+    A slot is a string ``content`` or one list part carrying a string ``text``;
+    images and other parts ride along untouched. Returns None unless the counts
+    line up exactly, so a rewrite never lands on the wrong slot.
+    """
+    if message_text_slot_count(message) != len(texts):
+        return None
+    content: Final = message.get("content")
+    if not isinstance(content, (str, list)):
+        return message
+    rewritten_content: Final = texts[0] if isinstance(content, str) else _content_with_slot_texts(content, texts)
+    rewritten: Final = {**message, "content": rewritten_content}  # mutable-ok: chat rows stay JSON-plain dicts
+    return cast("AllMessageValues", rewritten)  # cast-ok: the same row with only its text slots swapped
+
+
+class UnappliableRequestRewrite(Exception):
+    def __init__(self, guardrail_name: str) -> None:
+        super().__init__(
+            f"Guardrail '{guardrail_name}' rewrote the request in a way this endpoint cannot apply, "
+            "so the request was rejected rather than sent unrewritten"
+        )
+        self.guardrail_name: Final = guardrail_name
+
+
+def unappliable_request_rewrite(guardrail_name: str | None) -> UnappliableRequestRewrite:
+    return UnappliableRequestRewrite(guardrail_name or "unknown")

litellm/llms/bedrock/base_aws_llm.py

@@ -11,6 +11,7 @@ from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from functools import partial
 from threading import Lock
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, ParamSpec, TypeVar, cast, get_args, overload
 
 import httpx
@@ -96,6 +97,77 @@ def _assume_role_params(
             )
 
 
+_SecureTransportBool = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
+
+
+class _SecureTransportCondition(TypedDict):
+    Bool: ReadOnly[_SecureTransportBool]
+
+
+class _SessionPolicyStatement(TypedDict):
+    Sid: ReadOnly[str]
+    Effect: ReadOnly[Literal["Allow"]]
+    Action: ReadOnly[tuple[str, ...]]
+    Resource: ReadOnly[Literal["*"]]
+    Condition: ReadOnly[_SecureTransportCondition]
+
+
+class WebIdentitySessionPolicy(TypedDict):
+    Version: ReadOnly[Literal["2012-10-17"]]
+    Statement: ReadOnly[tuple[_SessionPolicyStatement, ...]]
+
+
+_WEB_IDENTITY_SESSION_POLICY_ACTIONS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
+    {
+        "BedrockLiteLLM": (
+            "bedrock:InvokeModel",
+            "bedrock:InvokeModelWithResponseStream",
+            "bedrock:CountTokens",
+            "bedrock:Rerank",
+            "bedrock:Retrieve",
+            "bedrock:ListKnowledgeBases",
+            "bedrock:InvokeAgent",
+            "bedrock:ApplyGuardrail",
+            "bedrock:GetGuardrail",
+            "bedrock:ListGuardrails",
+        ),
+        "BedrockAgentCoreLiteLLM": (
+            "bedrock-agentcore:InvokeAgentRuntime",
+            "bedrock-agentcore:InvokeAgentRuntimeForUser",
+            "bedrock-agentcore:InvokeGateway",
+        ),
+        "ClaudePlatformLiteLLM": (
+            "aws-external-anthropic:CreateInference",
+            "aws-external-anthropic:CreateBatchInference",
+            "aws-external-anthropic:CancelBatchInference",
+            "aws-external-anthropic:DeleteBatchInference",
+            "aws-external-anthropic:CountTokens",
+            "aws-external-anthropic:Get*",
+            "aws-external-anthropic:List*",
+        ),
+        "BedrockMantleLiteLLM": ("bedrock-mantle:CreateInference",),
+    }
+)
+
+_SECURE_TRANSPORT_ONLY: Final = _SecureTransportCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
+
+
+def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
+    return WebIdentitySessionPolicy(
+        Version="2012-10-17",
+        Statement=tuple(
+            _SessionPolicyStatement(
+                Sid=sid,
+                Effect="Allow",
+                Action=actions,
+                Resource="*",
+                Condition=_SECURE_TRANSPORT_ONLY,
+            )
+            for sid, actions in _WEB_IDENTITY_SESSION_POLICY_ACTIONS.items()
+        ),
+    )
+
+
 class BedrockRequestTarget(BaseModel):
     aws_region_name: str
     aws_bedrock_runtime_endpoint: str | None
@@ -940,60 +1012,12 @@ class BaseAWSLLM(SignsRequestsWithAWS):
         # auth only (static creds + IRSA take other code paths).
         # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
-        bedrock_session_policy: Final = {
-            "Version": "2012-10-17",
-            "Statement": [
-                {
-                    "Sid": "BedrockLiteLLM",
-                    "Effect": "Allow",
-                    "Action": [
-                        "bedrock:InvokeModel",
-                        "bedrock:InvokeModelWithResponseStream",
-                        "bedrock:CountTokens",
-                        "bedrock:ApplyGuardrail",
-                        "bedrock:GetGuardrail",
-                        "bedrock:ListGuardrails",
-                    ],
-                    "Resource": "*",
-                    "Condition": {"Bool": {"aws:SecureTransport": "true"}},
-                },
-                # Claude Platform on AWS (added by #27678 for the
-                # ``bedrock/claude_platform/<model>`` route) lives under
-                # a separate IAM action namespace; without these entries
-                # the OIDC path 403s on every claude_platform request
-                # even with a fully permissive identity policy (#30200).
-                {
-                    "Sid": "ClaudePlatformLiteLLM",
-                    "Effect": "Allow",
-                    "Action": [
-                        "aws-external-anthropic:CreateInference",
-                        "aws-external-anthropic:CreateBatchInference",
-                        "aws-external-anthropic:CancelBatchInference",
-                        "aws-external-anthropic:DeleteBatchInference",
-                        "aws-external-anthropic:CountTokens",
-                        "aws-external-anthropic:Get*",
-                        "aws-external-anthropic:List*",
-                    ],
-                    "Resource": "*",
-                    "Condition": {"Bool": {"aws:SecureTransport": "true"}},
-                },
-                {
-                    "Sid": "BedrockMantleLiteLLM",
-                    "Effect": "Allow",
-                    "Action": [
-                        "bedrock-mantle:CreateInference",
-                    ],
-                    "Resource": "*",
-                    "Condition": {"Bool": {"aws:SecureTransport": "true"}},
-                },
-            ],
-        }
         assume_role_params: Final = {
             "RoleArn": aws_role_name,
             "RoleSessionName": aws_session_name,
             "WebIdentityToken": oidc_token,
             "DurationSeconds": 3600,
-            "Policy": json.dumps(bedrock_session_policy, separators=(",", ":")),
+            "Policy": json.dumps(build_web_identity_session_policy(), separators=(",", ":")),
         }
 
         # Add ExternalId parameter if provided

litellm/llms/bedrock/claude_platform/messages_transformation.py

@@ -46,6 +46,7 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
         headers = self._update_headers_with_anthropic_beta(
             headers=headers,
             optional_params=optional_params,
+            messages=messages,
         )
 
         return headers, api_base

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -17,7 +17,7 @@ BaseAWSLLM._sign_request after the request body is finalized.
 
 import json
 from collections.abc import Mapping
-from typing import Any, Final
+from typing import Any, Final, cast  # noqa: TID251  # map_openai_params returns the filtered params as a bare dict
 
 import httpx
 from typing_extensions import ReadOnly, TypedDict
@@ -32,6 +32,7 @@ from litellm.llms.bedrock_mantle.common_utils import (
     BedrockMantleAuthMixin,
 )
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
     ResponseInputParam,
@@ -58,8 +59,6 @@ _BEDROCK_MANTLE_SUPPORTED_RESPONSE_TOOL_TYPES: Final = frozenset(
 _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 _BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"})
 
-_CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
-
 _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
 _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
 _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
@@ -233,17 +232,14 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         litellm_params: GenericLiteLLMParams,
         headers: dict,
     ) -> dict:
-        remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
-        normalized_input: Final = self._normalize_codex_input_items(remaining_input)
+        params: Final = cast(  # cast-ok: the base signature leaves the params dict untyped
+            "ResponsesAPIOptionalRequestParams", response_api_optional_request_params
+        )
+        hoisted: Final = hoist_additional_tools(input, params.get("tools"))
+        normalized_input: Final = self._normalize_codex_input_items(hoisted.input)
         request_params: Final = (
-            {
-                **response_api_optional_request_params,
-                "tools": [
-                    *(response_api_optional_request_params.get("tools") or []),
-                    *hoisted_tools,
-                ],
-            }
-            if hoisted_tools
+            self._params_with_hoisted_tools(params, hoisted)
+            if hoisted.hoisted
             else response_api_optional_request_params
         )
         return super().transform_responses_api_request(
@@ -254,41 +250,14 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
             headers=headers,
         )
 
-    @staticmethod
-    def _is_codex_additional_tools_item(item: Any) -> bool:
-        return isinstance(item, dict) and item.get("type") == _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
-
-    @staticmethod
-    def _tools_of_additional_tools_item(item: "dict[str, Any]") -> "list[Any]":
-        tools: Final = item.get("tools")
-        return tools if isinstance(tools, list) else []
-
     @classmethod
-    def _hoist_codex_additional_tools(
-        cls,
-        input: "str | ResponseInputParam",
-    ) -> "tuple[str | ResponseInputParam, list[Any]]":
-        """Codex's "responses lite" wire mode ships tool definitions inside
-        `input` as {"type": "additional_tools", "role": "developer",
-        "tools": [...]} items. api.openai.com accepts that item type; Mantle
-        rejects the whole request with 400 "Invalid 'input': value did not
-        match any expected variant" but accepts the same tools at the top
-        level, so move them there and strip the items from `input`.
-        """
-        if not isinstance(input, list):
-            return input, []
-        additional_tools_items: Final = [item for item in input if cls._is_codex_additional_tools_item(item)]
-        if not additional_tools_items:
-            return input, []
-        remaining_input: Final = [item for item in input if not cls._is_codex_additional_tools_item(item)]
-        hoisted_tools = [tool for item in additional_tools_items for tool in cls._tools_of_additional_tools_item(item)]
-        verbose_logger.debug(
-            "Bedrock Mantle Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) "
-            "into the top-level tools param (Mantle rejects that input item type).",
-            len(hoisted_tools),
-            len(additional_tools_items),
-        )
-        return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
+    def _params_with_hoisted_tools(
+        cls, params: Mapping[str, object], hoisted: HoistedAdditionalTools
+    ) -> dict[str, object]:
+        supported_tools: Final = cls._filter_unsupported_tools(list(hoisted.tools))
+        if supported_tools:
+            return {**params, "tools": supported_tools}
+        return {key: value for key, value in params.items() if key != "tools"}
 
     @staticmethod
     def _agent_message_text(item: "Mapping[str, object]") -> str:

litellm/llms/deepseek/messages/transformation.py

@@ -66,6 +66,7 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig):
             headers=headers,
             optional_params=optional_params,
             custom_llm_provider=self.custom_llm_provider or "deepseek",
+            messages=messages,
         )
 
         return headers, api_base

litellm/llms/gemini/realtime/transformation.py

@@ -115,6 +115,19 @@ def _parse_setup(session_configuration_request: str) -> BidiGenerateContentSetup
     return envelope.get("setup", empty_setup)
 
 
+def _grounding_metadata_from_frame(frame: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
+    """Read ``serverContent.groundingMetadata`` off the frame that carries the turn's usage.
+
+    Live reports grounding in the server frames rather than in ``usageMetadata``, and it emits both
+    on the same frame, so the per-query charge is countable at the point usage is built.
+    """
+    server_content: Final = frame.get("serverContent")
+    if not isinstance(server_content, Mapping):
+        return ()
+    metadata: Final = server_content.get("groundingMetadata")
+    return (metadata,) if isinstance(metadata, Mapping) else ()
+
+
 # Google bills Live transcription at an estimated 25 audio tokens/sec of input and
 # 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
 GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
@@ -323,7 +336,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                 )
             elif key == "input_audio_transcription" and value is not None:
                 optional_params["inputAudioTranscription"] = {}
-            elif key == "turn_detection":
+            elif key == "turn_detection" and value is not None:
                 value_typed = cast(OpenAIRealtimeTurnDetection, value)
                 if (
                     isinstance(value_typed, dict)
@@ -1049,6 +1062,11 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
                     {**cast(dict, message), "usageMetadata": resolved_usage_metadata},
                 ),
             )
+            grounding_metadata: Final = _grounding_metadata_from_frame(message)
+            if grounding_metadata:
+                VertexGeminiConfig._set_grounding_usage_counters(  # pyright: ignore[reportPrivateUsage]  # shared with the chat path; no public alias exists yet
+                    _chat_completion_usage, grounding_metadata
+                )
         else:
             _chat_completion_usage = get_empty_usage()
 

litellm/llms/github_copilot/messages/transformation.py

@@ -92,7 +92,7 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig):
             headers["anthropic-version"] = "2023-06-01"
 
         headers = self._update_headers_with_anthropic_beta(
-            headers, optional_params, custom_llm_provider="github_copilot"
+            headers, optional_params, custom_llm_provider="github_copilot", messages=messages
         )
 
         return headers, dynamic_api_base

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -42,6 +42,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     stream_item_field,
     stream_item_fingerprint,
     stream_item_items,
+    unappliable_request_rewrite,
 )
 from litellm.main import stream_chunk_builder
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@@ -196,6 +197,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             else:
                 # Step 3: Map guardrail responses back to original message structure
                 if guardrailed_texts and texts_to_check:
+                    if len(guardrailed_texts) != len(text_task_mappings):
+                        raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
                     await self._apply_guardrail_responses_to_input_texts(
                         messages=messages,
                         responses=guardrailed_texts,
@@ -210,6 +213,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                         task_mappings=tool_call_task_mappings,
                     )
 
+        elif (
+            not images_to_check
+            and not guardrail_to_apply.records_own_guardrail_information
+            and (not_run_reason := self._not_run_reason(messages)) is not None
+        ):
+            guardrail_to_apply.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_json_response=not_run_reason,
+                request_data=data,
+                guardrail_status="not_run",
+            )
+
         verbose_proxy_logger.debug(
             "OpenAI Chat Completions: Processed input messages: %s",
             data.get("messages"),
@@ -217,6 +231,28 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         return data
 
+    def _not_run_reason(
+        self,
+        messages: Sequence[dict[str, Any]],  # mutable-ok: raw request messages consumed by _extract_inputs
+    ) -> str | None:
+        """Why nothing was scanned, or None when the only unscoped content is images, which this handler never scans."""
+        texts: Final[list[str]] = []  # mutable-ok: filled by _extract_inputs
+        images: Final[list[str]] = []  # mutable-ok: filled by _extract_inputs
+        tool_calls: Final[list[ChatCompletionToolParam]] = []  # mutable-ok: filled by _extract_inputs
+        for msg_idx, message in enumerate(messages):
+            self._extract_inputs(
+                message=message,
+                msg_idx=msg_idx,
+                texts_to_check=texts,
+                images_to_check=images,
+                tool_calls_to_check=tool_calls,
+                text_task_mappings=[],  # mutable-ok: required by _extract_inputs, unused here
+                tool_call_task_mappings=[],  # mutable-ok: required by _extract_inputs, unused here
+            )
+        if texts or tool_calls:
+            return "no scannable content after message scoping"
+        return None if images else "no scannable content"
+
     def extract_request_tool_names(self, data: dict) -> list[str]:
         """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name)."""
         names: Final[list[str]] = []

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -56,6 +56,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     stream_item_field,
     stream_item_fingerprint,
     stream_item_items,
+    unappliable_request_rewrite,
 )
 from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
 from litellm.responses.litellm_completion_transformation.transformation import (
@@ -495,13 +496,13 @@ class OpenAIResponsesHandler(BaseTranslation):
                 data["instructions"] = written_back.instructions  # rebind-ok: data is an out-param
         elif isinstance(input_data, str):
             guardrailed_texts: Final = guardrailed_inputs.get("texts") or ()
+            if len(guardrailed_texts) > 1:
+                raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
             data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data  # rebind-ok: data is an out-param
         else:
             rewritten_texts: Final = guardrailed_inputs.get("texts") or ()
             if len(rewritten_texts) != len(extracted.task_mappings):
-                from litellm.proxy.policy_engine.pipeline_executor import UnappliableRequestRewrite
-
-                raise UnappliableRequestRewrite(guardrail_to_apply.guardrail_name or "unknown")
+                raise unappliable_request_rewrite(guardrail_to_apply.guardrail_name)
             await self._apply_guardrail_responses_to_input(
                 messages=input_data,
                 responses=rewritten_texts,

litellm/llms/openai/responses/guardrail_translation/tool_merge.py

@@ -6,8 +6,10 @@ from typing import Final, TypeAlias
 from pydantic import BaseModel, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
 from litellm.responses.litellm_completion_transformation.transformation import (
     NAMESPACE_DESCRIPTION_SEPARATOR,
+    NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS,
     LiteLLMCompletionResponsesConfig,
 )
 
@@ -34,8 +36,8 @@ def _validated_tools(values: Iterable[object]) -> tuple[Tool, ...]:
     return tuple(tool for tool in validated if tool is not None)
 
 
-def _is_function(tool: Tool) -> bool:
-    return tool.get("type") == "function"
+def _has_chat_tool(member: Tool) -> bool:
+    return member.get("type") in NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS
 
 
 def _chat_tool_key(tool: Tool) -> str:
@@ -67,18 +69,19 @@ def _function_fields(tool: Tool) -> Tool:
     return function if function is not None else MappingProxyType({})
 
 
-def _without_namespace_prefix(key: str, value: object, prefix: str) -> object:
-    if key != "description" or not isinstance(value, str) or not value.startswith(prefix):
+def _member_description(key: str, value: object, prefix: str, suffix: str) -> object:
+    if key != "description" or not isinstance(value, str):
         return value
-    return value[len(prefix) :]
+    return value.replace(prefix, "", 1).replace(suffix, "", 1)
 
 
 def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_description: str) -> Tool:
     flattened_function: Final = _function_fields(flattened)
     prefix: Final = f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}" if namespace_description else ""
+    suffix: Final = custom_tool_grammar_suffix(member.get("format")) if member.get("type") == "custom" else ""
     changed_function: Final = MappingProxyType(
         {
-            key: _without_namespace_prefix(key, value, prefix)
+            key: _member_description(key, value, prefix, suffix)
             for key, value in _function_fields(guardrailed).items()
             if flattened_function.get(key) != value
         }
@@ -93,8 +96,8 @@ def _rebuilt_member(member: Tool, flattened: Tool, guardrailed: Tool, namespace_
     return {**member, **changed_extras, **changed_function}  # mutable-ok: json.dumps rejects MappingProxyType
 
 
-def _rebuilt_function_members(
-    function_members: Sequence[Tool],
+def _rebuilt_flattened_members(
+    flattened_members: Sequence[Tool],
     flattened_group: Sequence[Tool],
     group_keys: Sequence[IndexedKey],
     guardrailed_by_key: Mapping[IndexedKey, Tool],
@@ -106,7 +109,7 @@ def _rebuilt_function_members(
         else member
         if guardrailed_by_key[key] == flattened
         else _rebuilt_member(member, flattened, guardrailed_by_key[key], namespace_description)
-        for member, flattened, key in zip(function_members, flattened_group, group_keys)
+        for member, flattened, key in zip(flattened_members, flattened_group, group_keys)
     )
 
 
@@ -118,9 +121,9 @@ def _rebuilt_namespace(
     guardrailed_by_key: Mapping[IndexedKey, Tool],
 ) -> tuple[Tool, ...]:
     namespace_description: Final = str(original.get("description") or "")
-    rebuilt_functions: Final = iter(
-        _rebuilt_function_members(
-            tuple(member for member in members if _is_function(member)),
+    rebuilt_flattened: Final = iter(
+        _rebuilt_flattened_members(
+            tuple(member for member in members if _has_chat_tool(member)),
             flattened_group,
             group_keys,
             guardrailed_by_key,
@@ -129,7 +132,7 @@ def _rebuilt_namespace(
     )
     rebuilt_members: Final = tuple(
         rebuilt
-        for rebuilt in (next(rebuilt_functions) if _is_function(member) else member for member in members)
+        for rebuilt in (next(rebuilt_flattened) if _has_chat_tool(member) else member for member in members)
         if rebuilt is not None
     )
     if not rebuilt_members:
@@ -149,7 +152,7 @@ def _merged_original(
     if guardrailed_group == tuple(flattened_group):
         return (original,)
     members: Final = _namespace_members(original) if original.get("type") == "namespace" else ()
-    if members and sum(map(_is_function, members)) == len(flattened_group):
+    if members and sum(map(_has_chat_tool, members)) == len(flattened_group):
         return _rebuilt_namespace(original, members, flattened_group, group_keys, guardrailed_by_key)
     if not guardrailed_group:
         return ()

litellm/llms/openai_like/messages/transformation.py

@@ -56,6 +56,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
         merged: Final = self._update_headers_with_anthropic_beta(
             headers=normalized,
             optional_params=optional_params,
+            messages=messages,
         )
         return merged, api_base
 

litellm/llms/vertex_ai/batches/transformation.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, Final
 from urllib.parse import unquote
 
@@ -8,7 +9,36 @@ from litellm.llms.vertex_ai.common_utils import (
 )
 from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest
 from litellm.types.llms.vertex_ai import *
-from litellm.types.utils import LiteLLMBatch
+from litellm.types.utils import LiteLLMBatch, PromptTokensDetailsWrapper
+
+
+def vertex_prompt_tokens_details(
+    usage_metadata: Mapping[str, object],
+) -> PromptTokensDetailsWrapper | None:
+    raw_details: Final = usage_metadata.get("promptTokensDetails")
+    if not isinstance(raw_details, list):
+        return None
+
+    def _normalize(detail: object) -> tuple[str, int] | None:
+        if not isinstance(detail, Mapping):
+            return None
+        modality: Final = detail.get("modality")
+        token_count: Final = detail.get("tokenCount")
+        if not isinstance(modality, str) or not isinstance(token_count, int):
+            return None
+        return modality.upper(), token_count
+
+    parsed_details: Final = tuple(_normalize(detail) for detail in raw_details)
+    normalized: Final = tuple(detail for detail in parsed_details if detail is not None)
+    if len(normalized) != len(parsed_details):
+        return None
+
+    return PromptTokensDetailsWrapper(
+        text_tokens=sum(token_count for modality, token_count in normalized if modality in ("TEXT", "DOCUMENT")),
+        audio_tokens=sum(token_count for modality, token_count in normalized if modality == "AUDIO"),
+        image_tokens=sum(token_count for modality, token_count in normalized if modality == "IMAGE"),
+        video_tokens=sum(token_count for modality, token_count in normalized if modality == "VIDEO"),
+    )
 
 
 class VertexAIBatchTransformation:

litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py

@@ -298,8 +298,6 @@ def transform_openai_input_gemini_embed_content(
 
 
 _IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"})
-_VIDEO_TOKENS_PER_SECOND: Final = 258.0
-_AUDIO_TOKENS_PER_SECOND: Final = 32.0
 _usage_metadata_adapter: Final = TypeAdapter(UsageMetadata)
 
 
@@ -339,11 +337,12 @@ def _is_image_element(
     return False
 
 
-def _count_input_images(
+def _is_image_only_input(
     input: GeminiEmbeddingInput,
     resolved_files: Mapping[str, Mapping[str, str]],
-) -> int:
-    return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files))
+) -> bool:
+    elements: Final = _flatten_input(input)
+    return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements)
 
 
 def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int:
@@ -372,30 +371,29 @@ def _usage_from_embed_content_response(
     total_tokens: Final = usage_metadata.get("totalTokenCount") or prompt_tokens
 
     details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or ()
+    if not details:
+        return Usage(
+            prompt_tokens=prompt_tokens,
+            total_tokens=total_tokens,
+            prompt_tokens_details=PromptTokensDetailsWrapper(
+                text_tokens=0,
+                image_tokens=prompt_tokens if _is_image_only_input(input, resolved_files) else 0,
+            ),
+        )
+
     text_tokens: Final = _tokens_for_modality(details, "TEXT")
     audio_tokens: Final = _tokens_for_modality(details, "AUDIO")
+    image_tokens: Final = _tokens_for_modality(details, "IMAGE")
     video_tokens: Final = _tokens_for_modality(details, "VIDEO")
-    image_count: Final = _count_input_images(input, resolved_files)
-
-    video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0
-    audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0
-
-    # generic_cost_per_token rewrites text_tokens to the full prompt minus
-    # other modalities when both text_tokens and image_count are zero. For
-    # video, that misallocates video tokens to text; a 1-token floor sidesteps
-    # the rewrite and keeps billing on input_cost_per_video_per_second.
-    needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0
-    resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens
 
     return Usage(
         prompt_tokens=prompt_tokens,
         total_tokens=total_tokens,
         prompt_tokens_details=PromptTokensDetailsWrapper(
-            text_tokens=resolved_text_tokens,
+            text_tokens=text_tokens,
             audio_tokens=audio_tokens,
-            image_count=image_count,
-            video_length_seconds=video_length_seconds,
-            audio_length_seconds=audio_length_seconds,
+            image_tokens=image_tokens,
+            video_tokens=video_tokens,
         ),
     )
 
@@ -415,8 +413,7 @@ def process_embed_content_response(
         model_response: EmbeddingResponse to populate
         model: Model name
         response_json: Raw JSON response from embedContent endpoint
-        resolved_files: Mapping of file references (files/abc) to {mime_type, uri},
-            used to bill resolved image references at the per-image rate
+        resolved_files: Mapping of file references to resolved metadata
 
     Returns:
         EmbeddingResponse with single embedding

litellm/model_prices_and_context_window_backup.json

@@ -25601,10 +25601,14 @@
         "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models"
     },
     "gemini-embedding-2-preview": {
-        "input_cost_per_audio_per_second": 0.00016,
-        "input_cost_per_image": 0.00012,
+        "input_cost_per_audio_token": 6.5e-06,
+        "input_cost_per_audio_token_batches": 3.25e-06,
+        "input_cost_per_image_token": 4.5e-07,
+        "input_cost_per_image_token_batches": 2.25e-07,
         "input_cost_per_token": 2e-07,
-        "input_cost_per_video_per_second": 0.00079,
+        "input_cost_per_token_batches": 1e-07,
+        "input_cost_per_video_token": 1.2e-05,
+        "input_cost_per_video_token_batches": 6e-06,
         "litellm_provider": "vertex_ai-embedding-models",
         "max_input_tokens": 8192,
         "max_tokens": 8192,
@@ -25615,13 +25619,14 @@
         "uses_embed_content": true
     },
     "gemini-embedding-2": {
-        "input_cost_per_audio_per_second": 0.00016,
         "input_cost_per_audio_token": 6.5e-06,
-        "input_cost_per_image": 0.00012,
+        "input_cost_per_audio_token_batches": 3.25e-06,
         "input_cost_per_image_token": 4.5e-07,
+        "input_cost_per_image_token_batches": 2.25e-07,
         "input_cost_per_token": 2e-07,
         "input_cost_per_token_batches": 1e-07,
-        "input_cost_per_video_per_second": 0.00079,
+        "input_cost_per_video_token": 1.2e-05,
+        "input_cost_per_video_token_batches": 6e-06,
         "litellm_provider": "vertex_ai-embedding-models",
         "max_input_tokens": 8192,
         "max_tokens": 8192,
@@ -25633,10 +25638,14 @@
         "uses_embed_content": true
     },
     "vertex_ai/gemini-embedding-2-preview": {
-        "input_cost_per_audio_per_second": 0.00016,
-        "input_cost_per_image": 0.00012,
+        "input_cost_per_audio_token": 6.5e-06,
+        "input_cost_per_audio_token_batches": 3.25e-06,
+        "input_cost_per_image_token": 4.5e-07,
+        "input_cost_per_image_token_batches": 2.25e-07,
         "input_cost_per_token": 2e-07,
-        "input_cost_per_video_per_second": 0.00079,
+        "input_cost_per_token_batches": 1e-07,
+        "input_cost_per_video_token": 1.2e-05,
+        "input_cost_per_video_token_batches": 6e-06,
         "litellm_provider": "vertex_ai",
         "max_input_tokens": 8192,
         "max_tokens": 8192,
@@ -25648,13 +25657,14 @@
         "uses_embed_content": true
     },
     "vertex_ai/gemini-embedding-2": {
-        "input_cost_per_audio_per_second": 0.00016,
         "input_cost_per_audio_token": 6.5e-06,
-        "input_cost_per_image": 0.00012,
+        "input_cost_per_audio_token_batches": 3.25e-06,
         "input_cost_per_image_token": 4.5e-07,
+        "input_cost_per_image_token_batches": 2.25e-07,
         "input_cost_per_token": 2e-07,
         "input_cost_per_token_batches": 1e-07,
-        "input_cost_per_video_per_second": 0.00079,
+        "input_cost_per_video_token": 1.2e-05,
+        "input_cost_per_video_token_batches": 6e-06,
         "litellm_provider": "vertex_ai",
         "max_input_tokens": 8192,
         "max_tokens": 8192,
@@ -25693,10 +25703,14 @@
     },
     "gemini/gemini-embedding-2-preview": {
         "deprecation_date": "2026-08-10",
-        "input_cost_per_audio_per_second": 0.00016,
-        "input_cost_per_image": 0.00012,
+        "input_cost_per_audio_token": 6.5e-06,
+        "input_cost_per_audio_token_batches": 3.25e-06,
+        "input_cost_per_image_token": 4.5e-07,
+        "input_cost_per_image_token_batches": 2.25e-07,
         "input_cost_per_token": 2e-07,
-        "input_cost_per_video_per_second": 0.00079,
+        "input_cost_per_token_batches": 1e-07,
+        "input_cost_per_video_token": 1.2e-05,
+        "input_cost_per_video_token_batches": 6e-06,
         "litellm_provider": "gemini",
         "max_input_tokens": 8192,
         "max_tokens": 8192,
@@ -25709,10 +25723,14 @@
         "tpm": 10000000
     },
     "gemini/gemini-embedding-2": {
-        "input_cost_per_audio_per_second": 0.00016,
-        "input_cost_per_image": 0.00012,
+        "input_cost_per_audio_token": 6.5e-06,
+        "input_cost_per_audio_token_batches": 3.25e-06,
+        "input_cost_per_image_token": 4.5e-07,
+        "input_cost_per_image_token_batches": 2.25e-07,
         "input_cost_per_token": 2e-07,
-        "input_cost_per_video_per_second": 0.00079,
+        "input_cost_per_token_batches": 1e-07,
+        "input_cost_per_video_token": 1.2e-05,
+        "input_cost_per_video_token_batches": 6e-06,
         "litellm_provider": "gemini",
         "max_input_tokens": 8192,
         "max_tokens": 8192,

litellm/models/credentials.py

@@ -5,6 +5,8 @@ These are the canonical credential types for the proxy. They live in the model
 layer; ``litellm.types.utils`` re-exports them for backwards compatibility.
 """
 
+from collections.abc import Mapping
+
 from pydantic import BaseModel, model_validator
 
 
@@ -27,3 +29,10 @@ class CreateCredentialItem(CredentialBase):
         if not values.get("credential_values") and not values.get("model_id"):
             raise ValueError("Either credential_values or model_id must be set")
         return values
+
+
+class UpdateCredentialItem(BaseModel):
+    credential_name: str
+    credential_info: Mapping[str, object]
+    credential_values: Mapping[str, object] | None = None
+    model_id: str | None = None

litellm/proxy/_lazy_openapi_snapshot.json

@@ -12968,18 +12968,24 @@
             "PHONE_NUMBER",
             "MEDICAL_LICENSE",
             "URL",
+            "MAC_ADDRESS",
+            "UUID",
             "US_BANK_NUMBER",
             "US_DRIVER_LICENSE",
             "US_ITIN",
             "US_PASSPORT",
             "US_SSN",
+            "US_MBI",
+            "US_NPI",
             "UK_NHS",
             "UK_NINO",
             "UK_PASSPORT",
             "UK_POSTCODE",
             "UK_VEHICLE_REGISTRATION",
+            "UK_DRIVING_LICENCE",
             "ES_NIF",
             "ES_NIE",
+            "ES_PASSPORT",
             "IT_FISCAL_CODE",
             "IT_DRIVER_LICENSE",
             "IT_VAT_CODE",
@@ -12997,7 +13003,38 @@
             "IN_VEHICLE_REGISTRATION",
             "IN_VOTER",
             "IN_PASSPORT",
-            "FI_PERSONAL_IDENTITY_CODE"
+            "IN_GSTIN",
+            "FI_PERSONAL_IDENTITY_CODE",
+            "DE_TAX_ID",
+            "DE_TAX_NUMBER",
+            "DE_VAT_ID",
+            "DE_PASSPORT",
+            "DE_ID_CARD",
+            "DE_FUEHRERSCHEIN",
+            "DE_SOCIAL_SECURITY",
+            "DE_HEALTH_INSURANCE",
+            "DE_LANR",
+            "DE_BSNR",
+            "DE_KFZ",
+            "DE_HANDELSREGISTER",
+            "DE_PLZ",
+            "KR_RRN",
+            "KR_FRN",
+            "KR_PASSPORT",
+            "KR_DRIVER_LICENSE",
+            "KR_BRN",
+            "CA_SIN",
+            "SE_PERSONNUMMER",
+            "SE_ORGANISATIONSNUMMER",
+            "TH_TNIN",
+            "TR_NATIONAL_ID",
+            "TR_LICENSE_PLATE",
+            "NG_NIN",
+            "NG_VEHICLE_REGISTRATION",
+            "PH_TIN",
+            "PH_UMID",
+            "PH_PASSPORT",
+            "ZA_ID_NUMBER"
           ],
           "title": "PiiEntityType",
           "type": "string"

litellm/proxy/_types.py

@@ -4,11 +4,12 @@ import os
 from collections.abc import Callable, Mapping
 from datetime import datetime
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple
+from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple
 
 import httpx
 from pydantic import (
     BaseModel,
+    BeforeValidator,
     ConfigDict,
     Field,
     Json,
@@ -47,6 +48,7 @@ from litellm.types.proxy.carried_budget_state import (
 )
 from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry
 from litellm.types.router import RouterErrors, UpdateRouterConfig
+from litellm.types.router_weights import validate_router_settings_dict
 from litellm.types.secret_managers.main import KeyManagementSystem
 from litellm.types.utils import (
     CallTypes,
@@ -284,6 +286,7 @@ class KeyManagementRoutes(str, enum.Enum):
     # team's `team_member_permissions`, non-admin members of that team may set
     # `access_group_ids` on keys they create/update. Default-deny.
     KEY_ACCESS_GROUP_ASSIGNMENT = "/key/access_group_assignment"
+    AUTO_ROUTER_MANAGE = "/auto_router/manage"
 
     # info and health routes
     KEY_INFO = "/key/info"
@@ -650,15 +653,18 @@ class LiteLLMRoutes(enum.Enum):
         KeyManagementRoutes.KEY_RESET_SPEND.value,
         KeyManagementRoutes.KEY_ALIASES.value,
         KeyManagementRoutes.KEY_ACCESS_GROUP_ASSIGNMENT.value,
+        KeyManagementRoutes.AUTO_ROUTER_MANAGE.value,
     ]
 
     management_routes = (
         [
             # user
             "/user/new",
+            "/management/v1/users/bulk",
             "/user/update",
             "/user/bulk_update",
             "/user/delete",
+            "/management/v1/users/bulk_delete",
             "/user/info",
             "/user/list",
             "/user/daily/activity",
@@ -838,6 +844,7 @@ class LiteLLMRoutes(enum.Enum):
     self_managed_routes = [
         "/team/member_add",
         "/team/member_delete",
+        "/management/v1/teams/{team_id}/members/bulk_delete",
         "/team/member_update",
         "/team/{team_id}/member/{user_id}/reset_spend",
         "/team/permissions_list",
@@ -864,6 +871,7 @@ class LiteLLMRoutes(enum.Enum):
         "/organization/daily/activity",
         "/user/available_roles",  # read-only role metadata; any authenticated user may read
         "/user/list",  # org admins checked in endpoint; non-admins get 403
+        "/management/v1/users/bulk_delete",  # proxy admins delete anyone, org admins only their orgs' users; others 403
         "/model/{model_id}/update",
         "/prompt/list",
         "/prompt/info",
@@ -1981,8 +1989,14 @@ class OrgMember(MemberBase):
 
 from litellm.models.team import TeamBase as TeamBase  # noqa: E402
 
+RouterSettingsDict = Annotated[
+    dict[str, object],
+    BeforeValidator(validate_router_settings_dict, json_schema_input_type=UpdateRouterConfig),
+]
+
 
 class NewTeamRequest(TeamBase):
+    router_settings: RouterSettingsDict | None = None
     model_aliases: dict | None = None
     tags: list | None = None
     guardrails: list[str] | None = None
@@ -2080,7 +2094,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
     allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None
     enforced_batch_output_expires_after: dict | None = None
     enforced_file_expires_after: dict | None = None
-    router_settings: dict | None = None
+    router_settings: RouterSettingsDict | None = None
     access_group_ids: list[str] | None = None
     budget_limits: list[BudgetLimitEntry] | None = None  # multiple concurrent budget windows
     default_team_member_models: list[str] | None = None  # default allowed_models seeded onto new team members

litellm/proxy/auth/auth_checks.py

@@ -39,6 +39,7 @@ from litellm.constants import (
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
+from litellm.models.project import LiteLLM_ProjectTable
 from litellm.proxy._types import (
     RBAC_ROLES,
     CallInfo,
@@ -109,7 +110,7 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics
 from litellm.repositories.budget_repository import BudgetRepository
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
 from litellm.repositories.organization_repository import OrganizationRepository
-from litellm.repositories.prisma_protocols import RowT_co
+from litellm.repositories.prisma_protocols import DatabaseClient, RowT_co
 from litellm.repositories.project_repository import ProjectRepository
 from litellm.repositories.table_repositories import (
     AccessGroupRepository,
@@ -847,6 +848,7 @@ BUDGET_ENFORCED_SIDE_EFFECT_ROUTES: Final = frozenset(
         "/health",
         "/health/services",
         "/health/test_connection",
+        "/auto_router/test_routing",
     }
 )
 
@@ -3172,7 +3174,7 @@ async def _delete_cache_access_object(
 @log_db_metrics
 async def get_access_object(
     access_group_id: str,
-    prisma_client: PrismaClient | None,
+    prisma_client: DatabaseClient | None,
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging | None = None,
 ) -> LiteLLM_AccessGroupTable:
@@ -3918,7 +3920,7 @@ async def get_org_object(
 async def _get_resources_from_access_groups(
     access_group_ids: Sequence[str],
     resource_field: Literal["access_model_names", "access_mcp_server_ids", "access_agent_ids"],
-    prisma_client: PrismaClient | None = None,
+    prisma_client: DatabaseClient | None = None,
     user_api_key_cache: UserApiKeyCache | None = None,
     proxy_logging_obj: ProxyLogging | None = None,
 ) -> list[str]:
@@ -3976,7 +3978,7 @@ async def _get_resources_from_access_groups(
 
 async def _get_models_from_access_groups(
     access_group_ids: Sequence[str],
-    prisma_client: PrismaClient | None = None,
+    prisma_client: DatabaseClient | None = None,
     user_api_key_cache: UserApiKeyCache | None = None,
     proxy_logging_obj: ProxyLogging | None = None,
 ) -> list[str]:
@@ -4475,6 +4477,7 @@ async def can_key_call_model(
     llm_model_list: Sequence[object] | None,
     valid_token: UserAPIKeyAuth,
     llm_router: litellm.Router | None,
+    prisma_client: DatabaseClient | None = None,
 ) -> Literal[True]:
     """
     Checks if token can call a given model
@@ -4504,6 +4507,7 @@ async def can_key_call_model(
         if key_access_group_ids:
             models_from_groups: Final = await _get_models_from_access_groups(
                 access_group_ids=key_access_group_ids,
+                prisma_client=prisma_client,
             )
             if models_from_groups:
                 return _can_object_call_model(
@@ -4632,6 +4636,7 @@ async def can_team_access_model(
     team_object: LiteLLM_TeamTable | None,
     llm_router: Router | None,
     team_model_aliases: dict[str, str] | None = None,
+    prisma_client: DatabaseClient | None = None,
 ) -> Literal[True]:
     """
     Returns True if the team can access a specific model.
@@ -4654,12 +4659,13 @@ async def can_team_access_model(
         if team_access_group_ids:
             models_from_groups: Final = await _get_models_from_access_groups(
                 access_group_ids=team_access_group_ids,
+                prisma_client=prisma_client,
             )
             if models_from_groups:
                 return _can_object_call_model(
                     model=model,
                     llm_router=llm_router,
-                    models=models_from_groups,
+                    models=list(dict.fromkeys([*(team_object.models if team_object else []), *models_from_groups])),
                     team_model_aliases=team_model_aliases,
                     team_id=team_object.team_id if team_object else None,
                     object_type="team",
@@ -4749,7 +4755,7 @@ async def _key_access_group_grants_model(
 
 def can_project_access_model(
     model: str | list[str],
-    project_object: LiteLLM_ProjectTableCachedObj,
+    project_object: LiteLLM_ProjectTable,
     llm_router: Router | None,
 ) -> Literal[True]:
     """

litellm/proxy/auth/auto_router_checks.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
+
+if TYPE_CHECKING:
+    from litellm.router import Router
+
+_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
+
+def _mapping(value: object) -> Mapping[str, object] | None:
+    try:
+        return _MAPPING_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+async def authorize_member_auto_router_inference(
+    *,
+    deployment: Mapping[str, object] | None,
+    request_kwargs: Mapping[str, object],
+    llm_router: Router,
+) -> None:
+    if deployment is None:
+        return
+    model_info: Final = _mapping(deployment.get("model_info"))
+    if model_info is None or model_info.get("member_auto_router") is not True:
+        return
+
+    from fastapi import HTTPException
+
+    from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
+    from litellm.proxy.auth.auth_checks import (
+        OrganizationNotFoundError,
+        TeamNotFoundError,
+        get_org_object,
+        get_project_object,
+        get_team_membership,
+        get_team_object,
+    )
+    from litellm.proxy.management_helpers.auto_router_permissions import (
+        MemberAutoRouterDependencyObjects,
+        authorize_member_auto_router_dependencies,
+        validate_member_auto_router_config,
+    )
+
+    metadata: Final = _mapping(request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)))
+    actor: Final = metadata.get("user_api_key_auth") if metadata is not None else None
+    team_id: Final = model_info.get("team_id")
+    if not isinstance(actor, UserAPIKeyAuth) or not isinstance(team_id, str) or not team_id:
+        raise HTTPException(status_code=403, detail="Member auto-routers require authenticated team access")
+    if actor.team_id != team_id and actor.user_role != LitellmUserRoles.PROXY_ADMIN:
+        raise HTTPException(status_code=403, detail="This auto-router belongs to a different team")
+
+    from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
+
+    if prisma_client is None:
+        raise HTTPException(status_code=503, detail="Cannot verify auto-router model access without a database")
+    try:
+        team: Final = await get_team_object(
+            team_id=team_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            parent_otel_span=actor.parent_otel_span,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except TeamNotFoundError as error:
+        raise HTTPException(status_code=403, detail="The auto-router team no longer exists") from error
+    if (
+        actor.user_role != LitellmUserRoles.PROXY_ADMIN
+        and actor.user_id is not None
+        and (not actor.user_id or not any(member.user_id == actor.user_id for member in team.members_with_roles))
+    ):
+        raise HTTPException(status_code=403, detail="You are no longer a member of this auto-router's team")
+    if team.blocked:
+        raise HTTPException(status_code=403, detail="This auto router's team is blocked.")
+    params: Final = _mapping(deployment.get("litellm_params"))
+    if params is None:
+        raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
+    raw_config: Final = _mapping(params.get("complexity_router_config"))
+    if raw_config is None:
+        raise HTTPException(status_code=403, detail="The member auto-router configuration is invalid")
+    default_model: Final = params.get("complexity_router_default_model")
+    config: Final = validate_member_auto_router_config(raw_config)
+    membership: Final = (
+        await get_team_membership(
+            user_id=actor.user_id,
+            team_id=team_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            parent_otel_span=actor.parent_otel_span,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+        if actor.user_id
+        else None
+    )
+    try:
+        organization: Final = (
+            await get_org_object(
+                org_id=team.organization_id,
+                prisma_client=prisma_client,
+                user_api_key_cache=user_api_key_cache,
+                parent_otel_span=actor.parent_otel_span,
+                proxy_logging_obj=proxy_logging_obj,
+            )
+            if team.organization_id
+            else None
+        )
+    except OrganizationNotFoundError as error:
+        raise HTTPException(status_code=403, detail="The auto router's organization is unavailable.") from error
+    project: Final = (
+        await get_project_object(
+            project_id=actor.project_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+        if actor.project_id
+        else None
+    )
+    await authorize_member_auto_router_dependencies(
+        config=config,
+        default_model=default_model if isinstance(default_model, str) else None,
+        user_api_key_dict=actor,
+        team=team,
+        prisma_client=None,
+        llm_router=llm_router,
+        dependency_objects=MemberAutoRouterDependencyObjects(
+            membership=membership, organization=organization, project=project
+        ),
+    )

litellm/proxy/auth/login_utils.py

@@ -249,6 +249,7 @@ async def authenticate_user(
 
         if os.getenv("DATABASE_URL") is not None:
             response = await generate_key_helper_fn(
+                llm_router=None,
                 request_type="key",
                 **{
                     "user_role": LitellmUserRoles.PROXY_ADMIN,
@@ -324,6 +325,7 @@ async def authenticate_user(
             await _rehash_password_if_needed(_user_row.user_id, password, _password)
             if os.getenv("DATABASE_URL") is not None:
                 response = await generate_key_helper_fn(
+                    llm_router=None,
                     request_type="key",
                     **{
                         "user_role": user_role,

litellm/proxy/auth/route_checks.py

@@ -1,5 +1,5 @@
 import re
-from collections.abc import Sequence
+from collections.abc import Collection
 from typing import Final
 
 from fastapi import HTTPException, Request, status
@@ -24,10 +24,13 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES: Final = frozenset(
     [
         # user
         "/user/new",
+        "/management/v1/users/bulk",
         "/user/delete",
+        "/management/v1/users/bulk_delete",
         "/user/bulk_update",
         # team
         "/team/new",
+        "/management/v1/teams/{team_id}/members/bulk_delete",
         "/team/update",
         "/team/delete",
         "/team/block",
@@ -587,7 +590,7 @@ class RouteChecks:
         return False
 
     @staticmethod
-    def check_route_access(route: str, allowed_routes: Sequence[str]) -> bool:
+    def check_route_access(route: str, allowed_routes: Collection[str]) -> bool:
         """
         Check if a route has access by checking both exact matches and patterns
 
@@ -758,9 +761,12 @@ class RouteChecks:
     _ADMIN_VIEWER_BLOCKED_WRITE_ROUTES = frozenset(
         [
             "/user/new",
+            "/management/v1/users/bulk",
             "/user/delete",
+            "/management/v1/users/bulk_delete",
             "/user/bulk_update",
             "/team/new",
+            "/management/v1/teams/{team_id}/members/bulk_delete",
             "/team/update",
             "/team/delete",
             "/model/new",
@@ -824,7 +830,7 @@ class RouteChecks:
                                 status_code=status.HTTP_403_FORBIDDEN,
                                 detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated",
                             )
-            elif route in _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES or (
+            elif RouteChecks.check_route_access(route=route, allowed_routes=_PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES) or (
                 route.startswith("/key/") and route.endswith(_PROXY_ADMIN_VIEW_ONLY_BLOCKED_KEY_SUFFIXES)
             ):
                 # Block write operations for PROXY_ADMIN_VIEW_ONLY
@@ -859,9 +865,9 @@ class RouteChecks:
         # Hard-block known write routes regardless of HTTP method (defensive
         # — these are POSTs in practice, but pinning them here protects
         # against future GET-shaped writes).
-        if route in RouteChecks._ADMIN_VIEWER_BLOCKED_WRITE_ROUTES or (
-            route.startswith("/key/") and route.endswith("/regenerate")
-        ):
+        if RouteChecks.check_route_access(
+            route=route, allowed_routes=RouteChecks._ADMIN_VIEWER_BLOCKED_WRITE_ROUTES
+        ) or (route.startswith("/key/") and route.endswith("/regenerate")):
             raise HTTPException(
                 status_code=status.HTTP_403_FORBIDDEN,
                 detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",

litellm/proxy/auth/user_api_key_auth.py

@@ -878,6 +878,7 @@ async def _auto_register_jwt_mapping(
     # the NOT NULL @id constraint. Every successful key-creation caller (e.g.
     # /key/generate) passes table_name="key" explicitly.
     key_data: Final = await generate_key_helper_fn(
+        llm_router=None,
         request_type="key",
         table_name="key",
         team_id=team_id,
@@ -2489,7 +2490,7 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc
 async def _run_centralized_common_checks(
     user_api_key_auth_obj: UserAPIKeyAuth,
     request: Request,
-    request_data: dict,
+    request_data: dict[str, object],
     route: str,
 ) -> None:
     """Run ``common_checks`` once at the ``user_api_key_auth`` wrapper

litellm/proxy/client/cli/README.md

@@ -580,8 +580,8 @@ What the command changed is recorded in `~/.litellm/claude_configure_state.json`
 `lite configure claude`, `lite login --config-claude`, `lite up` and `lite autoroute up` also install a status line (`~/.litellm/statusline.py`, registered as `statusLine` in `~/.claude/settings.json` unless you already run one) that shows which model the auto-router actually served the last turn and, once the proxy has recorded the session, what the session cost against the router's savings baseline:
 
 ```
-claude-auto · Routed to: claude-haiku-4-5  -63% vs Claude Opus 5
-LiteLLM       ████████░░░░░░░░░░░░░░░░  $0.14
+Routed to: claude-haiku-4-5  -63% vs Claude Opus 5
+claude-auto   ████████░░░░░░░░░░░░░░░░  $0.14
 Claude Opus 5 ████████████████████████  $0.38
 ```
 

litellm/proxy/client/cli/commands/pi.py

@@ -10,7 +10,7 @@ import os
 import tempfile
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass
-from enum import StrEnum
+from enum import Enum
 from pathlib import Path
 from types import MappingProxyType
 from typing import Annotated, Final
@@ -25,7 +25,7 @@ LITELLM_PROXY_API_KEY_ENV: Final = "LITELLM_PROXY_API_KEY"
 _REJECTED_STATUSES: Final = frozenset((401, 403))
 
 
-class ListingFailure(StrEnum):
+class ListingFailure(str, Enum):
     """Why a proxy could not be listed, decided once where the HTTP outcome is classified.
 
     `unreachable` means no response at all; the other kinds prove the proxy answered, so callers

litellm/proxy/client/cli/commands/statusline_script.py

@@ -28,6 +28,7 @@ import os
 import sys
 import tempfile
 import time
+import unicodedata
 import urllib.error
 import urllib.request
 from collections.abc import Callable, Mapping
@@ -42,7 +43,6 @@ FETCH_TIMEOUT_SECONDS: Final = 3
 BAR_WIDTH: Final = 24
 BAR_FULL: Final = "\u2588"
 BAR_EMPTY: Final = "\u2591"
-SEPARATOR: Final = " \u00b7 "
 TRANSCRIPT_SCAN_LIMIT_BYTES: Final = 4 * 1024 * 1024
 CLAUDE_BASE_URL_ENV_KEYS: Final = ("ANTHROPIC_BASE_URL",)
 CLAUDE_API_KEY_ENV_KEYS: Final = ("ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY")
@@ -50,7 +50,6 @@ CODEX_BASE_URL_ENV_KEYS: Final = ("OPENAI_BASE_URL",)
 CODEX_API_KEY_ENV_KEYS: Final = ("OPENAI_API_KEY",)
 CODEX_STOP_EVENT: Final = "Stop"
 SYNTHETIC_MODEL: Final = "<synthetic>"
-LITELLM_LABEL: Final = "LiteLLM"
 RESET: Final = "\033[0m"
 BOLD: Final = "\033[1m"
 DIM: Final = "\033[90m"
@@ -302,31 +301,37 @@ def _bar(fraction: float, color: str, width: int, use_color: bool) -> str:
     return f"{color}{BAR_FULL * filled}{DIM}{BAR_EMPTY * (width - filled)}{RESET}"
 
 
+def _display_width(label: str) -> int:
+    return sum(
+        2 if unicodedata.east_asian_width(character) in ("W", "F") else 1
+        for character in label
+        if unicodedata.category(character) not in ("Mn", "Me")
+    )
+
+
 def render(model: str, session: Session | None, config_dir: Path, use_color: bool, bar_width: int = BAR_WIDTH) -> str:
     def paint(code: str, text: str) -> str:
         return f"{code}{text}{RESET}" if use_color else text
 
     routed: Final = paint(BOLD, f"Routed to: {model}")
-    if session is None:
+    if session is None or session.baseline_model is None or session.baseline_spend <= 0:
         return routed
-    header: Final = f"{session.router_name}{SEPARATOR}{routed}"
-    if session.baseline_model is None or session.baseline_spend <= 0:
-        return header
     reference: Final = baseline_label(session.baseline_model, config_dir)
     pct: Final = (session.baseline_spend - session.spend) / session.baseline_spend * 100
     delta: Final = paint(LITELLM_COLOR, f"{'-' if pct >= 0 else '+'}{abs(round(pct))}% vs {reference}")
     peak: Final = max(session.spend, session.baseline_spend)
-    label_width: Final = max(len(LITELLM_LABEL), len(reference))
+    label_width: Final = max(_display_width(session.router_name), _display_width(reference))
     rows: Final = (
-        (LITELLM_LABEL, session.spend, LITELLM_COLOR),
+        (session.router_name, session.spend, LITELLM_COLOR),
         (reference, session.baseline_spend, BASELINE_COLOR),
     )
     lines: Final = (
-        f"{paint(DIM, label.ljust(label_width))} {_bar(amount / peak, color, bar_width, use_color)} "
+        f"{paint(DIM, label + ' ' * (label_width - _display_width(label)))} "
+        f"{_bar(amount / peak, color, bar_width, use_color)} "
         f"{paint(DIM, f'${amount:.2f}')}"
         for label, amount, color in rows
     )
-    return "\n".join((f"{header}  {delta}", *lines))
+    return "\n".join((f"{routed}  {delta}", *lines))
 
 
 def color_enabled(env: Mapping[str, str]) -> bool:

litellm/proxy/common_request_processing.py

@@ -14,6 +14,7 @@ import httpx
 import orjson
 from fastapi import HTTPException, Request, status
 from fastapi.responses import JSONResponse, Response, StreamingResponse
+from pydantic import ValidationError
 from starlette.types import Receive, Scope, Send
 
 import litellm
@@ -76,6 +77,7 @@ from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_di
 from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.router import RouterRateLimitError
+from litellm.types.router_weights import validate_router_weights
 
 _LateResponseT = TypeVar("_LateResponseT", bound=Response)
 _LlmCallT = TypeVar("_LlmCallT")
@@ -1939,6 +1941,13 @@ class ProxyBaseLLMRequestProcessing:
             # This avoids expensive Router instantiation on each request
             if router_settings is not None:
                 self.data["router_settings_override"] = router_settings
+                try:
+                    self.data["_router_weights"] = validate_router_weights(router_settings.get("weights"))
+                except ValidationError:
+                    self.data["_router_weights"] = None
+                    verbose_proxy_logger.warning(
+                        "Ignoring invalid saved router weights; update team/key router_settings"
+                    )
                 alias_target: Final = await _resolve_per_request_model_group_alias(
                     requested_model=self.data.get("model"),
                     router_settings=router_settings,

litellm/proxy/common_utils/encrypt_decrypt_utils.py

@@ -124,7 +124,7 @@ def decrypt_value_helper(
     key: str,  # this is just for debug purposes, showing the k,v pair that's invalid. not a signing key.
     exception_type: Literal["debug", "error"] = "error",
     return_original_value: bool = False,
-):
+) -> str | None:
     signing_key: Final = _get_salt_key()
 
     try:

litellm/proxy/common_utils/openai_error_payload.py

@@ -8,6 +8,8 @@ from typing import Final
 
 from fastapi import status
 
+from litellm.constants import STRINGIFIED_NONE
+
 _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
     {
         status.HTTP_401_UNAUTHORIZED: "authentication_error",
@@ -35,7 +37,7 @@ def openai_error_type(exc: object, status_code: int) -> str:
     """OpenAI types ``error.type`` as a required string, so an exception carrying none
     falls back to the type its status code stands for."""
     carried: Final = attribute_of(exc, "type")
-    if isinstance(carried, str):
+    if isinstance(carried, str) and carried != STRINGIFIED_NONE:
         return carried
     mapped: Final = _OPENAI_ERROR_TYPE_BY_STATUS.get(status_code)
     if mapped is not None:
@@ -49,4 +51,4 @@ def openai_error_param(exc: object) -> str | None:
     """OpenAI types ``error.param`` as nullable, so an exception carrying none
     serializes as JSON ``null``."""
     carried: Final = attribute_of(exc, "param")
-    return carried if isinstance(carried, str) else None
+    return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None

litellm/proxy/compliance_checks.py

@@ -26,7 +26,7 @@ class ComplianceChecker:
 
     def __init__(self, data: ComplianceCheckRequest):
         self.data = data
-        self.guardrails = data.guardrail_information or []
+        self.guardrails = tuple(g for g in data.guardrail_information or () if g.get("guardrail_status") != "not_run")
 
     def _get_guardrails_by_mode(self, mode: str) -> list[dict]:
         """

litellm/proxy/credential_endpoints/endpoints.py

@@ -2,25 +2,31 @@
 CRUD endpoints for storing reusable credentials.
 """
 
+from collections.abc import Mapping
 from typing import (
+    Annotated,
     Final,
     cast,  # noqa: TID251  # jsonify_object in proxy/utils.py is annotated with a bare dict
 )
 
 from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
+from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.litellm_logging import _get_masked_values
+from litellm.models.credentials import UpdateCredentialItem
 from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
 from litellm.proxy.utils import handle_exception_on_proxy, jsonify_object
+from litellm.repositories.base_repository import is_unique_violation
 from litellm.repositories.credentials_repository import CredentialsRepository
 from litellm.types.utils import CreateCredentialItem, CredentialItem
 
 router: Final = APIRouter()
+_CREDENTIAL_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
 
 
 class CredentialHelperUtils:
@@ -40,6 +46,33 @@ class CredentialHelperUtils:
         )
 
 
+def _credential_exists_detail(credential_name: str) -> str:
+    return (
+        f"Credential '{credential_name}' already exists. "
+        f"Update it with PATCH /credentials/{credential_name}, or delete it first."
+    )
+
+
+def get_llm_router() -> litellm.Router | None:
+    from litellm.proxy.proxy_server import llm_router
+
+    return llm_router
+
+
+def _resolve_deployment_credentials(llm_router: litellm.Router | None, model_id: str) -> Mapping[str, object]:
+    if llm_router is None:
+        raise HTTPException(
+            status_code=500,
+            detail="LLM router not found. Please ensure you have a valid router instance.",
+        )
+    if llm_router.get_deployment(model_id) is None:
+        raise HTTPException(status_code=404, detail="Model not found")
+    credential_values: Final = llm_router.get_deployment_credentials(model_id)
+    if credential_values is None:
+        raise HTTPException(status_code=404, detail="Model not found")
+    return _CREDENTIAL_DICT_ADAPTER.validate_python(credential_values)
+
+
 @router.post(
     "/credentials",
     dependencies=[Depends(user_api_key_auth)],
@@ -50,13 +83,14 @@ async def create_credential(
     fastapi_response: Response,
     credential: CreateCredentialItem,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    llm_router: Annotated[litellm.Router | None, Depends(get_llm_router)] = None,
 ):
     """
     [BETA] endpoint. This might change unexpectedly.
     Stores credential in DB.
     Reloads credentials in memory.
     """
-    from litellm.proxy.proxy_server import llm_router, prisma_client
+    from litellm.proxy.proxy_server import prisma_client
 
     try:
         if prisma_client is None:
@@ -64,29 +98,19 @@ async def create_credential(
                 status_code=500,
                 detail={"error": CommonProxyErrors.db_not_connected_error.value},
             )
-        if credential.model_id:
-            if llm_router is None:
-                raise HTTPException(
-                    status_code=500,
-                    detail="LLM router not found. Please ensure you have a valid router instance.",
-                )
-            # get model from router
-            model: Final = llm_router.get_deployment(credential.model_id)
-            if model is None:
-                raise HTTPException(status_code=404, detail="Model not found")
-            credential_values: Final = llm_router.get_deployment_credentials(credential.model_id)
-            if credential_values is None:
-                raise HTTPException(status_code=404, detail="Model not found")
-            credential.credential_values = credential_values
-
-        if credential.credential_values is None:
+        credential_values: Final = (
+            _resolve_deployment_credentials(llm_router, credential.model_id)
+            if credential.model_id
+            else credential.credential_values
+        )
+        if credential_values is None:
             raise HTTPException(
                 status_code=400,
                 detail="Credential values are required. Unable to infer credential values from model ID.",
             )
         processed_credential: Final = CredentialItem(
             credential_name=credential.credential_name,
-            credential_values=credential.credential_values,
+            credential_values=_CREDENTIAL_DICT_ADAPTER.validate_python(credential_values),
             credential_info=credential.credential_info,
         )
         encrypted_credential: Final = CredentialHelperUtils.encrypt_credential_values(processed_credential)
@@ -94,13 +118,18 @@ async def create_credential(
         credentials_dict_jsonified: Final = cast(  # cast-ok: deep-copies a model_dump, so keys are str
             "dict[str, object]", jsonify_object(credentials_dict)
         )
-        await CredentialsRepository(prisma_client).create(
-            data={
-                **credentials_dict_jsonified,
-                "created_by": user_api_key_dict.user_id,
-                "updated_by": user_api_key_dict.user_id,
-            }
-        )
+        try:
+            await CredentialsRepository(prisma_client).create(
+                data={
+                    **credentials_dict_jsonified,
+                    "created_by": user_api_key_dict.user_id,
+                    "updated_by": user_api_key_dict.user_id,
+                }
+            )
+        except Exception as e:
+            if not is_unique_violation(e):
+                raise
+            raise HTTPException(status_code=409, detail=_credential_exists_detail(credential.credential_name))
 
         ## ADD TO LITELLM ##
         CredentialAccessor.upsert_credentials([processed_credential])
@@ -300,9 +329,10 @@ def update_db_credential(
 async def update_credential(
     request: Request,
     fastapi_response: Response,
-    credential: CredentialItem,
+    credential: UpdateCredentialItem,
     credential_name: str = Path(..., description="The credential name, percent-decoded; may contain slashes"),
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    llm_router: Annotated[litellm.Router | None, Depends(get_llm_router)] = None,
 ):
     """
     [BETA] endpoint. This might change unexpectedly.
@@ -319,7 +349,16 @@ async def update_credential(
         db_credential: Final = await credentials_repository.find_by_name(credential_name)
         if db_credential is None:
             raise HTTPException(status_code=404, detail="Credential not found in DB.")
-        merged_credential: Final = update_db_credential(db_credential, credential)
+        patch: Final = CredentialItem(
+            credential_name=credential.credential_name,
+            credential_info=_CREDENTIAL_DICT_ADAPTER.validate_python(credential.credential_info),
+            credential_values=_CREDENTIAL_DICT_ADAPTER.validate_python(
+                _resolve_deployment_credentials(llm_router, credential.model_id)
+                if credential.model_id
+                else credential.credential_values or {}
+            ),
+        )
+        merged_credential: Final = update_db_credential(db_credential, patch)
         credential_object_jsonified: Final = cast(  # cast-ok: deep-copies a model_dump, so keys are str
             "dict[str, object]", jsonify_object(merged_credential.model_dump())
         )
@@ -341,11 +380,11 @@ async def update_credential(
 
         if existing_in_memory is not None:
             in_memory_values: Final = dict(existing_in_memory.credential_values or {})
-            if credential.credential_values:
-                in_memory_values.update(credential.credential_values)
+            if patch.credential_values:
+                in_memory_values.update(patch.credential_values)
             in_memory_info: Final = dict(existing_in_memory.credential_info or {})
-            if credential.credential_info:
-                in_memory_info.update(credential.credential_info)
+            if patch.credential_info:
+                in_memory_info.update(patch.credential_info)
             updated_in_memory: Final = CredentialItem(
                 credential_name=new_name,
                 credential_values=in_memory_values,

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py

@@ -7,7 +7,7 @@
 
 import fnmatch
 import os
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional
 
 import httpx
@@ -24,7 +24,7 @@ from litellm.llms.custom_httpx.http_handler import (
     httpxSpecialProvider,
 )
 from litellm.types.guardrails import GuardrailEventHooks
-from litellm.types.llms.openai import ChatCompletionToolParam
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
 from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
     GenericGuardrailAPIMetadata,
     GenericGuardrailAPIRequest,
@@ -150,6 +150,26 @@ def _extract_inbound_headers(
     return None
 
 
+def _structured_rows_to_write_back(
+    original_rows: Sequence[AllMessageValues] | None,
+    shown_rows: Sequence[AllMessageValues] | None,
+    returned_rows: Sequence[AllMessageValues],
+) -> tuple[AllMessageValues, ...] | None:
+    """The request model drops row keys its message types do not declare, so a
+    row the server echoes back verbatim is restored to the original row object.
+    A server that echoes every row back unchanged has not rewritten anything
+    per row, so its answer is read from texts, as it was before rows could be
+    returned at all."""
+    if original_rows is None or shown_rows is None or len(returned_rows) != len(original_rows):
+        return tuple(returned_rows)
+    if all(returned == shown for shown, returned in zip(shown_rows, returned_rows)):
+        return None
+    return tuple(
+        original if returned == shown else returned
+        for original, shown, returned in zip(original_rows, shown_rows, returned_rows)
+    )
+
+
 class GenericGuardrailAPI(CustomGuardrail):
     """
     Generic Guardrail API integration for LiteLLM.
@@ -322,6 +342,8 @@ class GenericGuardrailAPI(CustomGuardrail):
         texts: list,
         images: list[str] | None,
         tools: list[ChatCompletionToolParam] | None,
+        structured_messages: Sequence[AllMessageValues] | None,
+        shown_messages: Sequence[AllMessageValues] | None,
         guardrail_response: GenericGuardrailAPIResponse,
     ) -> GenericGuardrailAPIInputs:
         # Action is NONE or no modifications needed
@@ -336,6 +358,13 @@ class GenericGuardrailAPI(CustomGuardrail):
             return_inputs["tools"] = guardrail_response.tools
         elif tools:
             return_inputs["tools"] = tools
+        rows_to_write_back: Final = (
+            _structured_rows_to_write_back(structured_messages, shown_messages, guardrail_response.structured_messages)
+            if guardrail_response.structured_messages
+            else None
+        )
+        if rows_to_write_back is not None:
+            return_inputs["structured_messages"] = list(rows_to_write_back)  # mutable-ok: guardrail inputs take a list
         if guardrail_response.stream_holdback_chars is not None:
             return_inputs["stream_holdback_chars"] = guardrail_response.stream_holdback_chars
         return return_inputs
@@ -473,6 +502,8 @@ class GenericGuardrailAPI(CustomGuardrail):
                 texts=texts,
                 images=images,
                 tools=tools,
+                structured_messages=structured_messages,
+                shown_messages=guardrail_request.structured_messages,
                 guardrail_response=guardrail_response,
             )
 

litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py

@@ -1600,8 +1600,8 @@ class PanwPrismaAirsHandler(CustomGuardrail):
 
         Args:
             texts: Flattened text entries from the framework.
-            messages: Original request messages (request_data["messages"]),
-                      NOT structured_messages (which may have injected system content).
+            messages: The structured messages the framework flattened into ``texts``,
+                      hoisted top-level system prompt included, so positions line up.
 
         Returns a set of scannable indices, or None on count mismatch or no user/developer
         message (safety fallback to existing role-filter behavior).
@@ -1788,15 +1788,10 @@ class PanwPrismaAirsHandler(CustomGuardrail):
             structured_messages: Final = inputs.get("structured_messages")
             if structured_messages:
                 # For Anthropic /v1/messages: default to latest-user-only scanning.
-                # Uses request_data["messages"] (original format), NOT structured_messages
-                # (which has injected system content from adapter translation).
                 if self._use_latest_user_only(request_data, logging_obj):
-                    original_messages: Final = request_data.get("messages")
-                    if original_messages:
-                        scannable_indices = self._get_latest_user_text_indices(texts, original_messages)
+                    scannable_indices = self._get_latest_user_text_indices(texts, structured_messages)
                 # Fall through to existing role filtering if:
                 # - not Anthropic, OR flag explicitly False, OR
-                # - no original messages, OR
                 # - latest-user extraction returned None (no user / count mismatch)
                 if scannable_indices is None:
                     scannable_indices = self._get_scannable_text_indices(texts, structured_messages)

litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py

@@ -2,6 +2,7 @@ import asyncio
 import base64
 import os
 from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal, Optional
 
 import httpx
@@ -14,11 +15,13 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.llms.base_llm.guardrail_translation.utils import message_slot_texts, message_with_slot_texts
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
 from litellm.types.guardrails import GuardrailEventHooks
+from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import GenericGuardrailAPIInputs
 
 if TYPE_CHECKING:
@@ -28,12 +31,36 @@ if TYPE_CHECKING:
 
 _SANITIZE_FILE_FAIL_OPEN_TIMEOUT_SECONDS: Final = 30.0
 _SANITIZE_FILE_QUEUED_STATUSES: Final = frozenset({"created", "in progress"})
+_PROTECT_ROLES: Final = frozenset({"system", "user", "assistant"})
 
 
 class PromptSecurityGuardrailMissingSecrets(Exception):
     pass
 
 
+def _inputs_with_structured_messages(
+    inputs: GenericGuardrailAPIInputs, rewritten_messages: Sequence[AllMessageValues] | None
+) -> GenericGuardrailAPIInputs:
+    if rewritten_messages is None:
+        return inputs
+    patched: Final[GenericGuardrailAPIInputs] = {
+        **inputs,
+        "structured_messages": list(rewritten_messages),  # mutable-ok: the TypedDict field is declared as a list
+    }
+    return patched
+
+
+def _inputs_with_modifications(
+    inputs: GenericGuardrailAPIInputs,
+    modified_texts: list[str],
+    rewritten_messages: Sequence[AllMessageValues] | None,
+) -> GenericGuardrailAPIInputs:
+    if not modified_texts:
+        return _inputs_with_structured_messages(inputs, rewritten_messages)
+    with_texts: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": modified_texts}
+    return _inputs_with_structured_messages(with_texts, rewritten_messages)
+
+
 class _ProtectVerdict(TypedDict, total=False):
     """One side (``prompt`` or ``response``) of an ``/api/protect`` verdict."""
 
@@ -276,14 +303,39 @@ class PromptSecurityGuardrail(CustomGuardrail):
                 detail="Blocked by Prompt Security, Violations: " + ", ".join(violations),
             )
         elif action == "modify":
-            # Extract modified texts from modified_messages
             modified_messages: Final = result.get("modified_messages", [])
-            modified_texts: Final = self._extract_texts_from_messages(modified_messages)
-            if modified_texts:
-                inputs["texts"] = modified_texts
+            return _inputs_with_modifications(
+                inputs,
+                self._extract_texts_from_messages(modified_messages),
+                self._structured_messages_with_modifications(structured_messages, modified_messages),
+            )
 
         return inputs
 
+    def _is_sent_to_protect(self, message: Mapping[str, object]) -> bool:
+        return self.check_tool_results or message.get("role") in _PROTECT_ROLES
+
+    def _structured_messages_with_modifications(
+        self,
+        structured_messages: Sequence[AllMessageValues],
+        modified_messages: Sequence[Mapping[str, object]],
+    ) -> tuple[AllMessageValues, ...] | None:
+        sent_indices: Final = tuple(
+            index for index, message in enumerate(structured_messages) if self._is_sent_to_protect(message)
+        )
+        if not sent_indices or len(sent_indices) != len(modified_messages):
+            return None
+        rewritten: Final = tuple(
+            message_with_slot_texts(structured_messages[index], self._extract_texts_from_messages((modified,)))
+            for index, modified in zip(sent_indices, modified_messages)
+        )
+        replacements: Final = MappingProxyType(
+            {index: message for index, message in zip(sent_indices, rewritten) if message is not None}
+        )
+        if len(replacements) != len(sent_indices):
+            return None
+        return tuple(replacements.get(index, message) for index, message in enumerate(structured_messages))
+
     async def _apply_guardrail_on_response(
         self,
         inputs: GenericGuardrailAPIInputs,
@@ -347,19 +399,7 @@ class PromptSecurityGuardrail(CustomGuardrail):
         return inputs
 
     def _extract_texts_from_messages(self, messages: Sequence[Mapping[str, object]]) -> list[str]:
-        """Extract text content from messages."""
-        texts: Final = []
-        for message in messages:
-            content = message.get("content")
-            if isinstance(content, str):
-                texts.append(content)
-            elif isinstance(content, list):
-                for item in content:
-                    if isinstance(item, dict) and item.get("type") == "text":
-                        text = item.get("text")
-                        if text:
-                            texts.append(text)
-        return texts
+        return [text for message in messages for text in message_slot_texts(message)]
 
     async def _process_standalone_images(self, images: list[str], user_api_key_alias: str | None) -> None:
         """Process standalone images from inputs (data URLs)."""
@@ -681,14 +721,13 @@ class PromptSecurityGuardrail(CustomGuardrail):
 
         This allows checking tool results for indirect prompt injection when enabled.
         """
-        supported_roles: Final = ["system", "user", "assistant"]
         filtered_messages: Final = []
         transformed_count = 0
         filtered_count = 0
 
         for message in messages:
             role = message.get("role", "")
-            if role in supported_roles:
+            if role in _PROTECT_ROLES:
                 filtered_messages.append(message)
             else:
                 if self.check_tool_results:

litellm/proxy/guardrails/usage_endpoints.py

@@ -42,7 +42,7 @@ if TYPE_CHECKING:
 router: Final = APIRouter()
 
 _EMPTY_UNITS: Final[Mapping[str, int]] = MappingProxyType({})
-_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"passed": 0, "flagged": 1, "blocked": 2})
+_ACTION_SEVERITY: Final[Mapping[str, int]] = MappingProxyType({"not_run": 0, "passed": 1, "flagged": 2, "blocked": 3})
 
 _T = TypeVar("_T")
 
@@ -325,7 +325,7 @@ class UsageDetailResponse(BaseModel):
 class UsageLogEntry(BaseModel):
     id: str
     timestamp: str
-    action: str  # blocked | passed | flagged
+    action: str  # blocked | passed | flagged | not_run
     score: float | None
     latency_ms: float | None
     model: str | None

litellm/proxy/guardrails/usage_tracking.py

@@ -193,10 +193,12 @@ async def _upsert_rows_with_retry(
 
 
 def guardrail_status_to_action(status: str | None) -> str:
-    """Map StandardLogging guardrail_status to blocked/passed/flagged."""
+    """Map StandardLogging guardrail_status to blocked/passed/flagged/not_run."""
     if not status:
         return "passed"
     s: Final = (status or "").lower()
+    if s == "not_run":
+        return "not_run"
     if "intervened" in s or "block" in s:
         return "blocked"
     if "flagged" in s or "fail" in s or "error" in s:
@@ -354,37 +356,49 @@ async def process_spend_logs_guardrail_usage(
             "flagged_count": 0,
         }
     )
-    index_rows: Final[list[dict[str, object]]] = []
+    index_rows_by_key: Final[dict[tuple[str, str], dict[str, object]]] = {}
 
     for payload in logs_to_process:
         request_id = payload.get("request_id")
         start_time = _parse_payload_start_time(payload)
-        if not request_id or start_time is None:
+        if not isinstance(request_id, str) or not request_id or start_time is None:
             continue
         date_key = _date_str(start_time)
 
-        for entry in _parse_guardrail_info_from_payload(payload):
-            guardrail_id = entry.get("guardrail_id") or entry.get("guardrail_name") or ""
-            if not guardrail_id:
+        entries = _parse_guardrail_info_from_payload(payload)
+        ids_by_name = MappingProxyType(
+            {
+                e["guardrail_name"]: e["guardrail_id"]
+                for e in entries
+                if e.get("guardrail_id") and isinstance(e.get("guardrail_name"), str) and e["guardrail_name"]
+            }
+        )
+        for entry in entries:
+            raw_name = entry.get("guardrail_name")
+            guardrail_name = raw_name if isinstance(raw_name, str) else ""
+            guardrail_id = entry.get("guardrail_id") or ids_by_name.get(guardrail_name) or guardrail_name
+            if not isinstance(guardrail_id, str) or not guardrail_id:
                 continue
-            key = _MetricsKey(guardrail_id, date_key)
-            daily_guardrail[key]["requests_evaluated"] += 1
             action = guardrail_status_to_action(entry.get("guardrail_status"))
-            if action == "passed":
-                daily_guardrail[key]["passed_count"] += 1
-            elif action == "blocked":
-                daily_guardrail[key]["blocked_count"] += 1
-            else:
-                daily_guardrail[key]["flagged_count"] += 1
+            if action != "not_run":
+                key = _MetricsKey(guardrail_id, date_key)
+                daily_guardrail[key]["requests_evaluated"] += 1
+                if action == "passed":
+                    daily_guardrail[key]["passed_count"] += 1
+                elif action == "blocked":
+                    daily_guardrail[key]["blocked_count"] += 1
+                else:
+                    daily_guardrail[key]["flagged_count"] += 1
             policy_id = entry.get("policy_id")
-            index_rows.append(
-                {
+            prior = index_rows_by_key.get((request_id, guardrail_id))
+            if prior is None or (prior["policy_id"] is None and policy_id is not None):
+                index_rows_by_key[(request_id, guardrail_id)] = {
                     "request_id": request_id,
                     "guardrail_id": guardrail_id,
                     "policy_id": policy_id,
                     "start_time": start_time,
                 }
-            )
+    index_rows: Final = tuple(index_rows_by_key.values())
 
     async with pending.lock:
         pending_metrics: Final = pending.metrics

litellm/proxy/list_api/common.py

@@ -1,5 +1,6 @@
 """Contract machinery shared by every LiteLLM-defined list route, on any surface."""
 
+from collections.abc import Sequence
 from typing import Final
 from urllib.parse import urlencode
 
@@ -7,6 +8,7 @@ from fastapi import Request
 from fastapi.dependencies.utils import get_flat_params
 from fastapi.params import ParamTypes
 from fastapi.responses import JSONResponse
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.types.proxy.management_endpoints.management_v1 import (
     ListLinks,
@@ -56,6 +58,40 @@ def escape_like(value: str) -> str:
     return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
 
 
+class ValidationErrorDetail(TypedDict):
+    """The keys of a pydantic/FastAPI validation error a problem document needs."""
+
+    type: ReadOnly[str]
+    loc: ReadOnly[tuple[int | str, ...]]
+    msg: ReadOnly[str]
+
+
+def _is_length_error_of_rejected_items(error: ValidationErrorDetail, errors: Sequence[ValidationErrorDetail]) -> bool:
+    """pydantic counts only items that validated, so a bad item also trips the parent's min_length."""
+    return error["type"] == "too_short" and any(
+        len(other["loc"]) > len(error["loc"]) and other["loc"][: len(error["loc"])] == error["loc"] for other in errors
+    )
+
+
+def request_validation_problem(raw_errors: Sequence[ValidationErrorDetail]) -> ProblemDetail:
+    """A body that fails validation (an unknown field included) is 422; a bad query parameter is 400."""
+    errors: Final = tuple(error for error in raw_errors if not _is_length_error_of_rejected_items(error, raw_errors))
+    detail: Final = "; ".join(f"{'.'.join(str(part) for part in error['loc'][1:])}: {error['msg']}" for error in errors)
+    if any(error["loc"] and error["loc"][0] == "body" for error in errors):
+        return ProblemDetail(
+            type=f"{PROBLEM_TYPE_BASE}invalid-request-body",
+            title="Invalid request body",
+            status=422,
+            detail=detail or "The request body is invalid.",
+        )
+    return ProblemDetail(
+        type=f"{PROBLEM_TYPE_BASE}invalid-query-parameter",
+        title="Invalid query parameter",
+        status=400,
+        detail=detail or "The request query parameters are invalid.",
+    )
+
+
 def unknown_query_param_problem(unknown: tuple[str, ...], allowed: tuple[str, ...]) -> ProblemDetail:
     return ProblemDetail(
         type=f"{PROBLEM_TYPE_BASE}unknown-query-parameter",

litellm/proxy/litellm_pre_call_utils.py

@@ -221,6 +221,8 @@ LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
 )
 
 _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
+    "weights",
+    "_router_weights",
     "proxy_server_request",
     "standard_logging_object",
     "secret_fields",
@@ -334,7 +336,7 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg
 # and read by spend logs as fact; a client value has no legitimate meaning and no
 # key or team setting keeps it, so the strip is never gated.
 _ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
-    {"attempted_fallbacks", "original_model_group", CLIENT_OUTPUT_CEILING_METADATA_KEY}
+    {"attempted_fallbacks", "original_model_group", "request_retry_count", CLIENT_OUTPUT_CEILING_METADATA_KEY}
 )
 _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"
 

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -40,6 +40,14 @@ from litellm.proxy.litellm_pre_call_utils import (
     LiteLLMProxyRequestSetup,
     refresh_proxy_server_request_body_snapshot,
 )
+from litellm.proxy.management_endpoints.common_utils import (
+    _is_user_team_admin,  # pyright: ignore[reportPrivateUsage]  # shared owner of team-admin membership
+)
+from litellm.proxy.management_helpers.auto_router_permissions import (
+    authorize_member_auto_router_dependencies,
+    authorize_member_auto_router_team,
+    validate_member_auto_router_config,
+)
 from litellm.repositories.autorouter_session_repository import AutoRouterSessionRepository
 from litellm.repositories.base_repository import SupportsModelDump
 from litellm.repositories.team_repository import TeamRepository
@@ -72,13 +80,13 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
 )
 
 if TYPE_CHECKING:
-    from fastapi import APIRouter, Depends, HTTPException, Query, status
+    from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
 
     from litellm.proxy.utils import PrismaClient
     from litellm.router import Router
 else:
     try:
-        from fastapi import APIRouter, Depends, HTTPException, Query, status
+        from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
     except ImportError:
         # fastapi is only required for proxy, not for SDK usage
         pass
@@ -201,21 +209,14 @@ async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -
     return await prisma_client.db.query_raw(query, *args)
 
 
-async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> None:
-    """Allow exactly the callers who could create this router.
-
-    Both dry runs are gated like the write they rehearse rather than as reads: a proxy
-    admin, or a team admin naming their own team, matching /model/new. Routing a test
-    prompt can also spend money (an `llm` classifier config calls its classifier, a
-    semantic config embeds the prompt), so a read-level gate would be too loose anyway.
-    """
+async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> LiteLLM_TeamTable | None:
     from litellm.proxy.management_endpoints.model_management_endpoints import (
         ModelManagementAuthChecks,
     )
     from litellm.proxy.proxy_server import premium_user, prisma_client
 
     if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
-        return
+        return None
 
     if team_id is None:
         raise HTTPException(
@@ -244,12 +245,47 @@ async def _authorize_router_dry_run(user_api_key_dict: UserAPIKeyAuth, team_id:
             },
         )
 
-    ModelManagementAuthChecks.can_user_make_team_model_call(
-        team_id=team_id,
+    team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
+    if _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team):
+        ModelManagementAuthChecks.can_user_make_team_model_call(
+            team_id=team_id,
+            user_api_key_dict=user_api_key_dict,
+            team_obj=team,
+            premium_user=premium_user,
+        )
+        return None
+    authorize_member_auto_router_team(
         user_api_key_dict=user_api_key_dict,
-        team_obj=LiteLLM_TeamTable.model_validate(team_row.model_dump()),
+        team=team,
         premium_user=premium_user,
     )
+    return team
+
+
+async def _authorize_member_dry_run_config(
+    *,
+    config: Mapping[str, object],
+    default_model: str | None,
+    user_api_key_dict: UserAPIKeyAuth,
+    team: LiteLLM_TeamTable,
+) -> UserAPIKeyAuth:
+    from litellm.proxy.proxy_server import llm_router, prisma_client
+
+    if prisma_client is None or llm_router is None:
+        raise HTTPException(status_code=503, detail="Cannot verify auto-router model access")
+    validated: Final = validate_member_auto_router_config(config)
+    scoped_actor: Final = user_api_key_dict.model_copy(
+        update=MappingProxyType({"team_id": team.team_id, "team_models": team.models, "org_id": team.organization_id})
+    )
+    await authorize_member_auto_router_dependencies(
+        config=validated,
+        default_model=default_model,
+        user_api_key_dict=scoped_actor,
+        team=team,
+        prisma_client=prisma_client,
+        llm_router=llm_router,
+    )
+    return scoped_actor
 
 
 def _models_this_test_can_call(config: RequestComplexityRouterConfig) -> tuple[str, ...]:
@@ -326,16 +362,23 @@ async def validate_complexity_router_config(
 
     Runs the same check every write path runs (the router's own pydantic model), so a form can
     show the backend's exact verdict while the operator is still editing rather than after a
-    rejected save. Gated exactly like the save it rehearses: a proxy admin, or a team admin
-    naming their own team. Nothing is created, routed, or billed.
+    rejected save. Uses the same team opt-in and model-access checks as configuration
+    writes for members. Nothing is created, routed, or billed.
     """
-    await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
+    member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
 
     from litellm.router_utils.auto_router_model_naming import (
         validate_complexity_router_config_write,
     )
 
     error: Final = validate_complexity_router_config_write(data.complexity_router_config)
+    if error is None and member_team is not None:
+        await _authorize_member_dry_run_config(
+            config=data.complexity_router_config,
+            default_model=None,
+            user_api_key_dict=user_api_key_dict,
+            team=member_team,
+        )
     return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
 
 
@@ -349,6 +392,7 @@ async def validate_complexity_router_config(
 async def preview_auto_router_routing(
     data: AutoRouterRoutingTestRequest,
     user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    http_request: Request,
 ) -> AutoRouterRoutingTestResponse:
     """
     Route a single request through a complexity-router config and report where it landed.
@@ -392,7 +436,34 @@ async def preview_auto_router_routing(
     )
     from litellm.proxy.utils import get_available_models_for_user
 
-    await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
+    member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
+    actor: Final = (
+        await _authorize_member_dry_run_config(
+            config=data.complexity_router_config.model_dump(exclude_none=True),
+            default_model=data.default_model,
+            user_api_key_dict=user_api_key_dict,
+            team=member_team,
+        )
+        if member_team is not None
+        else user_api_key_dict
+    )
+    request_data: Final[dict[str, object]] = {  # mutable-ok: auth and routing enrich this request in place
+        **data.wire_body(),
+        "metadata": {},  # mutable-ok: centralized auth and identity stamping share this metadata bucket
+        "proxy_server_request": {"body": None},  # mutable-ok: the snapshot owner fills this body in place
+    }
+
+    if member_team is not None and _models_this_test_can_call(data.complexity_router_config):
+        from litellm.proxy.auth.user_api_key_auth import (
+            _run_centralized_common_checks,  # pyright: ignore[reportPrivateUsage]  # reuse the serving admission policy
+        )
+
+        await _run_centralized_common_checks(
+            user_api_key_auth_obj=actor,
+            request=http_request,
+            request_data=request_data,
+            route="/auto_router/test_routing",
+        )
 
     if llm_router is None:
         raise HTTPException(
@@ -404,7 +475,7 @@ async def preview_auto_router_routing(
 
     await _authorize_models_this_test_can_call(
         config=data.complexity_router_config,
-        user_api_key_dict=user_api_key_dict,
+        user_api_key_dict=actor,
         llm_router=llm_router,
     )
 
@@ -417,12 +488,8 @@ async def preview_auto_router_routing(
     )
 
     request_kwargs: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
-        data={  # mutable-ok: the request-metadata helper takes and returns request kwargs as a dict
-            **data.wire_body(),
-            "metadata": {},  # mutable-ok: the request-metadata helper writes the auth fields into this dict
-            "proxy_server_request": {"body": None},  # mutable-ok: the snapshot owner fills body in place
-        },
-        user_api_key_dict=user_api_key_dict,
+        data=request_data,
+        user_api_key_dict=actor,
         _metadata_variable_name="metadata",
     )
     refresh_proxy_server_request_body_snapshot(request_kwargs)

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -567,7 +567,7 @@ async def new_user(
             teams = check_if_default_team_set()
         organization_ids: Final = cast(list[str] | None, data_json.pop("organizations", None))
 
-        response: Final = await generate_key_helper_fn(request_type="user", **data_json)
+        response: Final = await generate_key_helper_fn(request_type="user", **data_json, llm_router=None)
         # Admin UI Logic
         # Add User to Team and Organization
         # if team_id passed add this user to the team

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -96,6 +96,7 @@ from litellm.proxy.management_endpoints.common_utils import (
 from litellm.proxy.management_endpoints.model_management_endpoints import (
     _add_model_to_db,
 )
+from litellm.proxy.management_endpoints.router_weights import validate_router_settings_weights
 from litellm.proxy.management_helpers.access_group_key_sync import (
     sync_key_access_group_membership,
     sync_key_regeneration_access_group_membership,
@@ -148,6 +149,7 @@ from litellm.types.proxy.management_endpoints.key_management_endpoints import (
     BulkUpdateKeyRequest,
     BulkUpdateKeyResponse,
     BulkUpdateTeamKeysRequest,
+    CustomKeyPolicyRequest,
     FailedKeyUpdate,
     KeySearchWhere,
     SuccessfulKeyUpdate,
@@ -201,6 +203,10 @@ class _KeyUpdateResult(TypedDict):
     data: ReadOnly[Mapping[str, object]]
 
 
+class _StoredKeyRouterSettings(BaseModel):
+    router_settings: Mapping[str, object] | None = None
+
+
 class _KeyRowWhere(TypedDict):
     token: ReadOnly[str]
 
@@ -280,6 +286,7 @@ def _config_table(prisma_client: PrismaClient) -> _ConfigTableActions:
 class _CustomKeyHooksModule(Protocol):
     user_custom_key_generate: Callable[..., Awaitable[Mapping[str, object]]] | None
     user_custom_key_update: Callable[..., Awaitable[Mapping[str, object]]] | None
+    user_custom_key_policy: Callable[..., Awaitable[Mapping[str, object]]] | None
 
 
 def _custom_key_generate_hook(
@@ -294,6 +301,161 @@ def _custom_key_update_hook(
     return hooks.user_custom_key_update
 
 
+def _custom_key_policy_hook(
+    hooks: _CustomKeyHooksModule,
+) -> Callable[..., Awaitable[Mapping[str, object]]] | None:
+    return hooks.user_custom_key_policy
+
+
+async def _enforce_custom_key_update_policy(
+    hook: Callable[..., Awaitable[Mapping[str, object]]] | None,
+    data: UpdateKeyRequest,
+) -> None:
+    if hook is None:
+        return
+    if not inspect.iscoroutinefunction(hook):
+        raise ValueError("user_custom_key_update must be a coroutine")
+    result: Final = await hook(data)
+    if not result.get("decision", True):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=result.get("message", "Authentication Failed - Custom Auth Rule"),
+        )
+
+
+async def _enforce_custom_key_policy(
+    hook: Callable[..., Awaitable[Mapping[str, object]]] | None,
+    build_policy_request: Callable[[], CustomKeyPolicyRequest],
+) -> None:
+    if hook is None:
+        return
+    if not inspect.iscoroutinefunction(hook):
+        raise ValueError("user_custom_key_policy must be a coroutine")
+    result: Final = await hook(build_policy_request())
+    if not result.get("decision", True):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail=result.get("message", "Authentication Failed - Custom Auth Rule"),
+        )
+
+
+_KEY_UPDATE_JSON_STRING_COLUMNS: Final = frozenset({"router_settings", "budget_limits"})
+
+_KEY_METADATA_REQUEST_FIELDS: Final = frozenset(
+    (*LiteLLM_ManagementEndpoint_MetadataFields_Premium, *LiteLLM_ManagementEndpoint_MetadataFields)
+)
+
+
+def _decode_json_string_column(column: str, value: object) -> object:
+    if column in _KEY_UPDATE_JSON_STRING_COLUMNS and isinstance(value, str):
+        return json.loads(value)
+    return value
+
+
+def _verification_token_from_row(row: Mapping[str, object]) -> LiteLLM_VerificationToken:
+    org_id: Final = row["organization_id"] if "organization_id" in row else row.get("org_id")
+    return LiteLLM_VerificationToken.model_validate(MappingProxyType({**row, "org_id": org_id}))
+
+
+def _effective_key_after_update(
+    existing_key_row: LiteLLM_VerificationToken,
+    non_default_values: Mapping[str, object],
+) -> LiteLLM_VerificationToken:
+    overlay: Final = MappingProxyType(
+        {column: _decode_json_string_column(column, value) for column, value in non_default_values.items()}
+    )
+    return _verification_token_from_row(
+        MappingProxyType({**existing_key_row.model_dump(), **overlay, "object_permission": None})
+    )
+
+
+def _update_policy_request(
+    operation: Literal["update", "regenerate"],
+    existing_key_row: LiteLLM_VerificationToken,
+    non_default_values: Mapping[str, object],
+    request: UpdateKeyRequest | RegenerateKeyRequest,
+) -> CustomKeyPolicyRequest:
+    return CustomKeyPolicyRequest(
+        operation=operation,
+        existing_key=_verification_token_from_row(existing_key_row.model_dump()),
+        effective_key=_effective_key_after_update(
+            existing_key_row=existing_key_row, non_default_values=non_default_values
+        ),
+        request=request,
+    )
+
+
+def _generate_budget_windows(
+    budget_limits: Sequence[BudgetLimitEntry] | None,
+) -> tuple[Mapping[str, object], ...] | None:
+    if not budget_limits:
+        return None
+    return tuple(
+        MappingProxyType(
+            {
+                **window.model_dump(),
+                "reset_at": get_budget_reset_time(budget_duration=window.budget_duration).isoformat(),
+            }
+        )
+        for window in budget_limits
+    )
+
+
+def _effective_key_for_generate(data: GenerateKeyRequest, now: datetime) -> LiteLLM_VerificationToken:
+    requested: Final = data.model_dump(exclude_unset=True, exclude_none=True)
+    metadata_fields: Final = MappingProxyType(
+        {field: value for field, value in requested.items() if field in _KEY_METADATA_REQUEST_FIELDS}
+    )
+    column_fields: Final = MappingProxyType(
+        {field: value for field, value in requested.items() if field not in _KEY_METADATA_REQUEST_FIELDS}
+    )
+    metadata: Final = data.metadata or MappingProxyType({})
+    folded_metadata: Final = {**metadata, **metadata_fields}  # mutable-ok: encrypt_callback_vars needs a dict
+    columns: Final = handle_key_type(data, {**column_fields})  # mutable-ok: handle_key_type mutates in place
+    expires: Final = (
+        now + timedelta(seconds=duration_in_seconds(duration=data.duration)) if data.duration is not None else None
+    )
+    budget_reset_at: Final = (
+        get_budget_reset_time(budget_duration=data.budget_duration) if data.budget_duration is not None else None
+    )
+    key_rotation_at: Final = (
+        now + timedelta(seconds=duration_in_seconds(duration=data.rotation_interval))
+        if data.auto_rotate and data.rotation_interval
+        else None
+    )
+    return _verification_token_from_row(
+        MappingProxyType(
+            {
+                **columns,
+                "metadata": encrypt_callback_vars(folded_metadata),
+                "expires": expires,
+                "budget_reset_at": budget_reset_at,
+                "key_rotation_at": key_rotation_at,
+                "budget_limits": _generate_budget_windows(data.budget_limits),
+                "object_permission": None,
+            }
+        )
+    )
+
+
+_EMPTY_DURATION_MEANS_UNCHANGED: Final = frozenset({"duration", "budget_duration"})
+
+
+def _regenerate_request_as_update_request(key: str, data: RegenerateKeyRequest) -> UpdateKeyRequest | None:
+    changed_fields: Final = MappingProxyType(
+        {
+            field: value
+            for field, value in data.model_dump(exclude_unset=True).items()
+            if field in UpdateKeyRequest.model_fields
+            and field != "key"
+            and not (field in _EMPTY_DURATION_MEANS_UNCHANGED and value == "")
+        }
+    )
+    if not changed_fields:
+        return None
+    return UpdateKeyRequest(key=key, **changed_fields)
+
+
 class _LegacyDumpable(Protocol):
     def dict(self) -> Mapping[str, object]: ...
 
@@ -987,6 +1149,7 @@ async def _common_key_generation_helper(
     litellm_changed_by: str | None,
     team_table: LiteLLM_TeamTableCachedObj | None,
 ) -> GenerateKeyResponse:
+    from litellm.proxy import proxy_server
     from litellm.proxy.proxy_server import (
         litellm_proxy_admin_name,
         llm_router,
@@ -1135,6 +1298,16 @@ async def _common_key_generation_helper(
             "litellm.proxy.proxy_server.generate_key_fn(): Enterprise key management params not applied - %s", e
         )
 
+    await _enforce_custom_key_policy(
+        hook=_custom_key_policy_hook(proxy_server),
+        build_policy_request=lambda: CustomKeyPolicyRequest(
+            operation="generate",
+            existing_key=None,
+            effective_key=_effective_key_for_generate(data=data, now=datetime.now(timezone.utc)),
+            request=data,
+        ),
+    )
+
     # TODO: @ishaan-jaff: Migrate all budget tracking to use LiteLLM_BudgetTable
     _budget_id = data.budget_id
     if prisma_client is not None and data.soft_budget is not None:
@@ -1330,7 +1503,7 @@ async def _common_key_generation_helper(
                 prisma_client=prisma_client,
             )
 
-    response = await generate_key_helper_fn(request_type="key", **data_json, table_name="key")
+    response = await generate_key_helper_fn(request_type="key", **data_json, table_name="key", llm_router=llm_router)
 
     response["soft_budget"] = data.soft_budget  # include the user-input soft budget in the response
 
@@ -2234,7 +2407,26 @@ async def _update_key_row_with_soft_budget(
 async def prepare_key_update_data(
     data: UpdateKeyRequest | RegenerateKeyRequest,
     existing_key_row: LiteLLM_VerificationToken,
+    *,
+    prisma_client: PrismaClient | None = None,
+    llm_router: Router | None = None,
 ):
+    if data.router_settings is not None or (
+        "router_settings" not in data.model_fields_set
+        and "team_id" in data.model_fields_set
+        and data.team_id != existing_key_row.team_id
+    ):
+        effective_settings: Final = (
+            data.router_settings
+            if data.router_settings is not None
+            else _StoredKeyRouterSettings.model_validate(existing_key_row, from_attributes=True).router_settings
+        )
+        await validate_router_settings_weights(
+            effective_settings,
+            team_id=data.team_id if "team_id" in data.model_fields_set else existing_key_row.team_id,
+            prisma_client=prisma_client,
+            llm_router=llm_router,
+        )
     data_json: Final[dict] = data.model_dump(exclude_unset=True)
     data_json.pop("key", None)
     data_json.pop("new_key", None)
@@ -2301,12 +2493,6 @@ async def prepare_key_update_data(
             # sentinel for Json? columns, so store the JSON literal null
             non_default_values["budget_limits"] = json.dumps(None)
 
-    if "object_permission" in non_default_values:
-        non_default_values = await _handle_update_object_permission(
-            data_json=non_default_values,
-            existing_key_row=existing_key_row,
-        )
-
     _metadata: Final = existing_key_row.metadata or {}
 
     # validate model_max_budget
@@ -2327,13 +2513,12 @@ async def prepare_key_update_data(
 async def _handle_update_object_permission(
     data_json: dict,
     existing_key_row: LiteLLM_VerificationToken,
+    prisma_client: PrismaClient,
 ) -> dict:
-    """
-    Handle the update of object permission.
-    """
-    from litellm.proxy.proxy_server import prisma_client
+    """Persist the requested object permission row and swap it for its id, only after the key policy allowed the write."""
+    if "object_permission" not in data_json:
+        return data_json
 
-    # Use the common helper to handle the object permission update
     object_permission_id: Final = await handle_update_object_permission_common(
         data_json=data_json,
         existing_object_permission_id=existing_key_row.object_permission_id,
@@ -2467,6 +2652,7 @@ async def _process_single_key_update(
     llm_router: Router | None,
     user_custom_key_update: Callable | None = None,
     existing_key_row: LiteLLM_VerificationToken | None = None,
+    user_custom_key_policy: Callable[..., Awaitable[Mapping[str, object]]] | None = None,
 ) -> dict[str, object]:
     """
     Process a single key update with all validations and checks.
@@ -2575,7 +2761,19 @@ async def _process_single_key_update(
         )
 
     # Prepare update data
-    non_default_values = await prepare_key_update_data(data=update_key_request, existing_key_row=existing_key_row)
+    non_default_values = await prepare_key_update_data(
+        data=update_key_request, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router
+    )
+
+    await _enforce_custom_key_policy(
+        hook=user_custom_key_policy,
+        build_policy_request=lambda: _update_policy_request(
+            operation="update",
+            existing_key_row=existing_key_row,
+            non_default_values=non_default_values,
+            request=update_key_request,
+        ),
+    )
 
     # Update key in database
     if prisma_client is None:
@@ -2584,7 +2782,12 @@ async def _process_single_key_update(
             detail={"error": "Database not connected"},
         )
 
-    _data: Final = {**non_default_values, "token": update_key_request.key}
+    update_values: Final = await _handle_update_object_permission(
+        data_json=non_default_values,
+        existing_key_row=existing_key_row,
+        prisma_client=prisma_client,
+    )
+    _data: Final = {**update_values, "token": update_key_request.key}
     response: Final[Mapping[str, object] | None] = cast(  # cast-ok: every update_data branch returns a str-keyed dict
         "Mapping[str, object] | None",
         await prisma_client.update_data(token=update_key_request.key, data=_data),
@@ -3077,23 +3280,13 @@ async def update_key_fn(
             user_api_key_cache=user_api_key_cache,
         )
 
-        # Custom key update hook
-        custom_key_update_hook: Final[Callable[..., Awaitable[Mapping[str, object]]] | None] = _custom_key_update_hook(
-            proxy_server
-        )
-        if custom_key_update_hook is not None:
-            if inspect.iscoroutinefunction(custom_key_update_hook):
-                result: Final = await custom_key_update_hook(data)
-            else:
-                raise ValueError("user_custom_key_update must be a coroutine")
-            decision: Final = result.get("decision", True)
-            message: Final = result.get("message", "Authentication Failed - Custom Auth Rule")
-            if not decision:
-                raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=message)
+        await _enforce_custom_key_update_policy(hook=_custom_key_update_hook(proxy_server), data=data)
 
         # Enforce upperbound key params on update (don't fill defaults)
         _enforce_upperbound_key_params(data, fill_defaults=False)
-        non_default_values: Final = await prepare_key_update_data(data=data, existing_key_row=existing_key_row)
+        non_default_values: Final = await prepare_key_update_data(
+            data=data, existing_key_row=existing_key_row, prisma_client=prisma_client, llm_router=llm_router
+        )
 
         # Only validate key_alias format if it's actually being changed
         new_key_alias: Final = non_default_values.get("key_alias", None)
@@ -3114,21 +3307,36 @@ async def update_key_fn(
             existing_key_alias=existing_key_row.key_alias,
         )
 
+        await _enforce_custom_key_policy(
+            hook=_custom_key_policy_hook(proxy_server),
+            build_policy_request=lambda: _update_policy_request(
+                operation="update",
+                existing_key_row=existing_key_row,
+                non_default_values=non_default_values,
+                request=data,
+            ),
+        )
+
         if prisma_client is None:
             raise Exception("Not connected to DB!")
 
+        update_values: Final = await _handle_update_object_permission(
+            data_json=non_default_values,
+            existing_key_row=existing_key_row,
+            prisma_client=prisma_client,
+        )
         changed_by: Final = user_api_key_dict.user_id or litellm_proxy_admin_name
         response: Final = (
             await _update_key_row_with_soft_budget(
                 prisma_client=prisma_client,
                 key=key,
                 data=data,
-                non_default_values=non_default_values,
+                non_default_values=update_values,
                 existing_key_row=existing_key_row,
                 changed_by=changed_by,
             )
             if "soft_budget" in data.model_fields_set
-            else await prisma_client.update_data(token=key, data=MappingProxyType({**non_default_values, "token": key}))
+            else await prisma_client.update_data(token=key, data=MappingProxyType({**update_values, "token": key}))
         )
 
         # Delete - key from cache, since it's been updated!
@@ -3263,6 +3471,7 @@ async def bulk_update_keys(
     )
 
     custom_key_update_hook: Final = _custom_key_update_hook(proxy_server)
+    custom_key_policy_hook: Final = _custom_key_policy_hook(proxy_server)
 
     if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value:
         raise HTTPException(
@@ -3310,6 +3519,7 @@ async def bulk_update_keys(
                 proxy_logging_obj=proxy_logging_obj,
                 llm_router=llm_router,
                 user_custom_key_update=custom_key_update_hook,
+                user_custom_key_policy=custom_key_policy_hook,
             )
 
             successful_updates.append(
@@ -3427,6 +3637,7 @@ async def bulk_update_team_keys(
     )
 
     custom_key_update_hook: Final = _custom_key_update_hook(proxy_server)
+    custom_key_policy_hook: Final = _custom_key_policy_hook(proxy_server)
 
     if prisma_client is None:
         raise HTTPException(
@@ -3557,6 +3768,7 @@ async def bulk_update_team_keys(
                 proxy_logging_obj=proxy_logging_obj,
                 llm_router=llm_router,
                 user_custom_key_update=custom_key_update_hook,
+                user_custom_key_policy=custom_key_policy_hook,
                 existing_key_row=existing_by_token[db_token],
             )
 
@@ -4082,6 +4294,40 @@ def _check_model_access_group(models: list[str] | None, llm_router: Router | Non
     return True
 
 
+_NO_METADATA: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def metadata_json_with_limits(
+    metadata: Mapping[str, object] | None,
+    *,
+    model_rpm_limit: Mapping[str, object] | None,
+    model_tpm_limit: Mapping[str, object] | None,
+    mcp_rpm_limit: Mapping[str, int] | None,
+    tag_rpm_limit: Mapping[str, int] | None,
+    guardrails: Sequence[str] | None,
+    policies: Sequence[str] | None,
+    prompts: Sequence[str] | None,
+) -> str:
+    """Serialize the stored metadata blob with the per-model, MCP, tag, guardrail, policy and prompt settings folded in."""
+    limits: Final = tuple(
+        (name, value)
+        for name, value in (
+            ("model_rpm_limit", model_rpm_limit),
+            ("model_tpm_limit", model_tpm_limit),
+            ("mcp_rpm_limit", mcp_rpm_limit),
+            ("tag_rpm_limit", tag_rpm_limit),
+            ("guardrails", guardrails),
+            ("policies", policies),
+            ("prompts", prompts),
+        )
+        if value is not None
+    )
+    if metadata is None and not limits:
+        return json.dumps(None)
+    merged: Final = {**(metadata or _NO_METADATA), **dict(limits)}  # mutable-ok: encrypt_callback_vars takes a dict
+    return json.dumps(encrypt_callback_vars(merged))
+
+
 async def generate_key_helper_fn(
     request_type: Literal["user", "key"],  # identifies if this request is from /user/new or /key/generate
     duration: str | None = None,
@@ -4137,15 +4383,24 @@ async def generate_key_helper_fn(
     object_permission: LiteLLM_ObjectPermissionBase | None = None,
     auto_rotate: bool | None = None,
     rotation_interval: str | None = None,
-    router_settings: dict | None = None,
+    router_settings: dict[str, object] | None = None,
     access_group_ids: list[str] | None = None,
     budget_limits: list | None = None,  # multiple concurrent budget windows
+    *,
+    llm_router: Router | None = None,
 ):
     from litellm.proxy.proxy_server import premium_user, prisma_client
 
     if prisma_client is None:
         raise Exception("Connect Proxy to database to generate keys - https://docs.litellm.ai/docs/proxy/virtual_keys ")
 
+    await validate_router_settings_weights(
+        router_settings,
+        team_id=team_id,
+        prisma_client=prisma_client,
+        llm_router=llm_router,
+    )
+
     if token is None:
         if key is not None:
             token = key
@@ -4184,31 +4439,16 @@ async def generate_key_helper_fn(
     permissions_json: Final = json.dumps(permissions)
     router_settings_json: Final = safe_dumps(router_settings) if router_settings is not None else safe_dumps({})
 
-    # Add model_rpm_limit and model_tpm_limit to metadata
-    if model_rpm_limit is not None:
-        metadata = metadata or {}
-        metadata["model_rpm_limit"] = model_rpm_limit
-    if model_tpm_limit is not None:
-        metadata = metadata or {}
-        metadata["model_tpm_limit"] = model_tpm_limit
-    if mcp_rpm_limit is not None:
-        metadata = metadata or {}
-        metadata["mcp_rpm_limit"] = mcp_rpm_limit
-    if tag_rpm_limit is not None:
-        metadata = metadata or {}
-        metadata["tag_rpm_limit"] = tag_rpm_limit
-    if guardrails is not None:
-        metadata = metadata or {}
-        metadata["guardrails"] = guardrails
-    if policies is not None:
-        metadata = metadata or {}
-        metadata["policies"] = policies
-    if prompts is not None:
-        metadata = metadata or {}
-        metadata["prompts"] = prompts
-
-    metadata = encrypt_callback_vars(metadata)
-    metadata_json: Final = json.dumps(metadata)
+    metadata_json: Final = metadata_json_with_limits(
+        metadata,
+        model_rpm_limit=model_rpm_limit,
+        model_tpm_limit=model_tpm_limit,
+        mcp_rpm_limit=mcp_rpm_limit,
+        tag_rpm_limit=tag_rpm_limit,
+        guardrails=guardrails,
+        policies=policies,
+        prompts=prompts,
+    )
     validate_model_max_budget(model_max_budget)
     model_max_budget_json: Final = json.dumps(model_max_budget)
     budget_fallbacks_json: Final = json.dumps(budget_fallbacks or {})
@@ -5070,6 +5310,7 @@ async def _insert_deprecated_key(
 async def _execute_virtual_key_regeneration(
     *,
     prisma_client: PrismaClient,
+    llm_router: Router | None = None,
     key_in_db: LiteLLM_VerificationToken,
     hashed_api_key: str,
     key: str,
@@ -5080,6 +5321,7 @@ async def _execute_virtual_key_regeneration(
     proxy_logging_obj: ProxyLogging,
 ) -> GenerateKeyResponse:
     """Generate new token, update DB, invalidate cache, and return response."""
+    from litellm.proxy import proxy_server
     from litellm.proxy.proxy_server import hash_token
 
     # Mirror the /key/update ownership rebind guard. See helper docstring.
@@ -5127,15 +5369,34 @@ async def _execute_virtual_key_regeneration(
 
     non_default_values = {}
     if data is not None:
+        update_request: Final = _regenerate_request_as_update_request(key=hashed_api_key, data=data)
+        if update_request is not None:
+            await _enforce_custom_key_update_policy(hook=_custom_key_update_hook(proxy_server), data=update_request)
         # Enforce upperbound key params on regenerate (don't fill defaults)
         _enforce_upperbound_key_params(data, fill_defaults=False)
-        non_default_values = await prepare_key_update_data(data=data, existing_key_row=key_in_db)
+        non_default_values = await prepare_key_update_data(
+            data=data, existing_key_row=key_in_db, prisma_client=prisma_client, llm_router=llm_router
+        )
         # Only validate key_alias format if it's actually being changed
         new_key_alias: Final = non_default_values.get("key_alias")
         if new_key_alias != key_in_db.key_alias:
             _validate_key_alias_format(key_alias=new_key_alias)
         verbose_proxy_logger.debug("non_default_values: %s", non_default_values)
-    update_data.update(non_default_values)
+    await _enforce_custom_key_policy(
+        hook=_custom_key_policy_hook(proxy_server),
+        build_policy_request=lambda: _update_policy_request(
+            operation="regenerate",
+            existing_key_row=key_in_db,
+            non_default_values=non_default_values,
+            request=data if data is not None else RegenerateKeyRequest(),
+        ),
+    )
+    update_values: Final = await _handle_update_object_permission(
+        data_json=non_default_values,
+        existing_key_row=key_in_db,
+        prisma_client=prisma_client,
+    )
+    update_data.update(update_values)
     jsonified_update_data: Final[Mapping[str, object]] = prisma_client.jsonify_object(data=update_data)
 
     # Snapshot before the token update: the FK cascade rewrites mapping rows to the new hash,
@@ -5145,6 +5406,13 @@ async def _execute_virtual_key_regeneration(
         prisma_client=prisma_client,
     )
 
+    await _persist_deleted_verification_tokens(
+        keys=[key_in_db],
+        prisma_client=prisma_client,
+        user_api_key_dict=user_api_key_dict,
+        litellm_changed_by=litellm_changed_by,
+    )
+
     # If grace period set, insert deprecated key so old key remains valid
     await _insert_deprecated_key(
         prisma_client=prisma_client,
@@ -5268,6 +5536,7 @@ async def regenerate_key_fn(
     try:
         from litellm.proxy.proxy_server import (
             hash_token,
+            llm_router,
             master_key,
             premium_user,
             prisma_client,
@@ -5443,19 +5712,9 @@ async def regenerate_key_fn(
         if litellm_changed_by is not None and not isinstance(litellm_changed_by, str):
             litellm_changed_by = None
 
-        # Save the old key record to deleted table before regeneration.
-        # This preserves key_alias and team_id metadata for historical spend records.
-        # If this fails, abort the regeneration to avoid permanently losing the
-        # old hash→metadata mapping.
-        await _persist_deleted_verification_tokens(
-            keys=[_key_in_db],
-            prisma_client=prisma_client,
-            user_api_key_dict=user_api_key_dict,
-            litellm_changed_by=litellm_changed_by,
-        )
-
         return await _execute_virtual_key_regeneration(
             prisma_client=prisma_client,
+            llm_router=llm_router,
             key_in_db=_key_in_db,
             hashed_api_key=hashed_api_key,
             key=key,

litellm/proxy/management_endpoints/management_v1/__init__.py

@@ -10,9 +10,17 @@ from litellm.proxy.management_endpoints.management_v1.budgets import (
 from litellm.proxy.management_endpoints.management_v1.spend_logs import (
     router as spend_logs_router,
 )
+from litellm.proxy.management_endpoints.management_v1.teams import (
+    router as teams_router,
+)
+from litellm.proxy.management_endpoints.management_v1.users import (
+    router as users_router,
+)
 
 router: Final = APIRouter()
 router.include_router(budgets_router)
 router.include_router(spend_logs_router)
+router.include_router(teams_router)
+router.include_router(users_router)
 
 __all__ = ["router"]

litellm/proxy/management_endpoints/management_v1/teams.py

@@ -0,0 +1,94 @@
+"""`POST /management/v1/teams/{team_id}/members/bulk_delete`."""
+
+from typing import Annotated, Final
+
+from fastapi import APIRouter, Depends
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem, reject_unknown_query_params
+from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
+from litellm.proxy.management_helpers.bulk_user_deletion import bulk_remove_team_members
+from litellm.proxy.management_helpers.utils import (
+    management_endpoint_wrapper,  # pyright: ignore[reportUnknownVariableType]  # legacy decorator is untyped
+)
+from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail
+from litellm.types.proxy.management_endpoints.team_endpoints import (
+    BulkTeamMemberDeleteRequest,
+    BulkTeamMemberDeleteResponse,
+)
+
+router: Final = APIRouter(prefix=MANAGEMENT_V1_PREFIX)
+
+
+@router.post(
+    "/teams/{team_id}/members/bulk_delete",
+    tags=["team management"],  # mutable-ok: FastAPI types `tags` as list[str], not Sequence
+    dependencies=(Depends(user_api_key_auth), Depends(reject_unknown_query_params)),
+    response_model=BulkTeamMemberDeleteResponse,
+)
+@management_endpoint_wrapper
+async def bulk_delete_team_members_action(
+    team_id: str,
+    data: BulkTeamMemberDeleteRequest,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+) -> BulkTeamMemberDeleteResponse:
+    """
+    Remove up to 500 members from one team in one call. Same authorization as
+    `/team/member_delete`: proxy admins, the team's admins, and admins of the team's
+    organization. Each member is named by exactly one of `user_id` or `user_email`;
+    unknown body fields are a 422 and an unknown team is a 404.
+
+    `data` holds one result per requested member, in request order. A row is
+    `success: false` with an `error` when it names nobody on the team or repeats an
+    earlier row. The roster is rewritten once, under the team's advisory lock, so a
+    concurrent member_add is never overwritten from a stale read.
+
+    Example curl:
+    ```
+    curl --location 'http://0.0.0.0:4000/management/v1/teams/team-1/members/bulk_delete' \
+        --header 'Authorization: Bearer sk-1234' \
+        --header 'Content-Type: application/json' \
+        --data '{"members": [{"user_id": "user-1"}, {"user_email": "user-2@example.com"}]}'
+    ```
+    """
+    try:
+        from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
+
+        if prisma_client is None:
+            raise ManagementProblem(
+                ProblemDetail(
+                    type=f"{PROBLEM_TYPE_BASE}database-not-connected",
+                    title="Database not connected",
+                    status=503,
+                    detail=CommonProxyErrors.db_not_connected_error.value,
+                )
+            )
+
+        results: Final = await bulk_remove_team_members(
+            team_id=team_id,
+            data=data,
+            user_api_key_dict=user_api_key_dict,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+        return BulkTeamMemberDeleteResponse(data=results)
+
+    except ManagementProblem:
+        raise
+    except Exception as e:  # noqa: BLE001  # a driver error answers as a problem document, not the OpenAI error shape
+        verbose_proxy_logger.exception(
+            "litellm.proxy.management_endpoints.management_v1.teams.bulk_delete_team_members_action(): "
+            "Exception occured - %s",
+            e,
+        )
+        raise ManagementProblem(
+            ProblemDetail(
+                type=f"{PROBLEM_TYPE_BASE}internal-server-error",
+                title="Internal server error",
+                status=500,
+                detail="Failed to remove team members.",
+            )
+        )

litellm/proxy/management_endpoints/management_v1/users.py

@@ -0,0 +1,187 @@
+"""`POST /management/v1/users/bulk` and `POST /management/v1/users/bulk_delete`."""
+
+from typing import Annotated, Final
+
+from fastapi import APIRouter, Depends, Header
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem, reject_unknown_query_params
+from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
+from litellm.proxy.management_helpers.bulk_user_creation import bulk_create_users
+from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users
+from litellm.proxy.management_helpers.utils import (
+    management_endpoint_wrapper,  # pyright: ignore[reportUnknownVariableType]  # legacy untyped decorator
+)
+from litellm.types.proxy.management_endpoints.internal_user_endpoints import (
+    BulkDeleteUserRequest,
+    BulkDeleteUsersResponse,
+    BulkNewUserRequest,
+    BulkNewUserResponse,
+)
+from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail
+
+router: Final = APIRouter(prefix=MANAGEMENT_V1_PREFIX)
+
+
+@router.post(
+    "/users/bulk",
+    tags=["Internal User management"],  # mutable-ok: fastapi types tags as list[str | Enum]
+    dependencies=(Depends(user_api_key_auth),),
+    response_model=BulkNewUserResponse,
+)
+@management_endpoint_wrapper
+async def bulk_create_users_route(
+    data: BulkNewUserRequest,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+) -> BulkNewUserResponse:
+    """
+    Create up to 500 internal users in one request, optionally adding each one to teams.
+
+    Every entry in `users` takes the same fields as `/user/new`, with two differences: `auto_create_key`
+    defaults to `false` (opt in per user to also get a virtual key back) and `send_invite_email` is not
+    supported. Unknown fields are rejected with 422. Rows are validated together (duplicate ids or emails,
+    unknown teams, roles the caller may not grant), inserted in one statement, and each referenced team is
+    written once for all of its new members.
+
+    Rows fail independently: a bad row is reported in `data` with `success: false` and an `error`, and the
+    other rows still get created. A user that was created but could not be added to one of its teams is
+    reported with `success: true`, `teams` listing where they did land, and `error` naming the failed team.
+    The whole request is refused with a 403 problem document only if creating the valid rows would exceed
+    the license seat limit.
+
+    Example curl:
+    ```
+    curl -X POST "http://localhost:4000/management/v1/users/bulk" \\
+    -H "Content-Type: application/json" \\
+    -H "Authorization: Bearer sk-1234" \\
+    -d '{
+        "users": [
+            {"user_email": "a@example.com", "user_role": "internal_user", "teams": ["team-1"]},
+            {"user_email": "b@example.com", "user_role": "internal_user", "auto_create_key": true}
+        ]
+    }'
+    ```
+
+    Returns `data` (one entry per input row, in order, with `user_id`, `user_email`, `success`, `teams`,
+    `key`, `error`) and `meta` with `total_requested`, `created` and `failed`.
+    """
+    try:
+        from litellm.proxy.proxy_server import (
+            _license_check,  # pyright: ignore[reportPrivateUsage]  # same proxy license singleton /user/new reads
+            litellm_proxy_admin_name,
+            prisma_client,
+            user_api_key_cache,
+        )
+
+        if prisma_client is None:
+            raise ManagementProblem(
+                ProblemDetail(
+                    type=f"{PROBLEM_TYPE_BASE}database-not-connected",
+                    title="Database not connected",
+                    status=503,
+                    detail=CommonProxyErrors.db_not_connected_error.value,
+                )
+            )
+
+        return await bulk_create_users(
+            users=data.users,
+            user_api_key_dict=user_api_key_dict,
+            prisma_client=prisma_client,
+            license_check=_license_check,
+            litellm_proxy_admin_name=litellm_proxy_admin_name,
+            user_api_key_cache=user_api_key_cache,
+        )
+
+    except ManagementProblem:
+        raise
+    except Exception:  # noqa: BLE001  # a driver error answers as a problem document, not the OpenAI error shape
+        verbose_proxy_logger.exception("/management/v1/users/bulk: Exception occurred")
+        raise ManagementProblem(
+            ProblemDetail(
+                type=f"{PROBLEM_TYPE_BASE}internal-server-error",
+                title="Internal server error",
+                status=500,
+                detail="Failed to create users.",
+            )
+        )
+
+
+@router.post(
+    "/users/bulk_delete",
+    tags=["Internal User management"],  # mutable-ok: FastAPI types `tags` as list[str], not Sequence
+    dependencies=(Depends(user_api_key_auth), Depends(reject_unknown_query_params)),
+    response_model=BulkDeleteUsersResponse,
+)
+@management_endpoint_wrapper
+async def bulk_delete_users_action(
+    data: BulkDeleteUserRequest,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    litellm_changed_by: Annotated[
+        str | None,
+        Header(description="Who the caller is acting for; recorded on the audit log entries this call writes."),
+    ] = None,
+) -> BulkDeleteUsersResponse:
+    """
+    Delete up to 500 users in one call, taking each out of every team it belongs to.
+    Same authorization as `/user/delete`: proxy admins may delete anyone, org admins
+    only users inside organizations they administer. Unknown body fields are a 422.
+
+    `data` holds one result per requested `user_id`, in request order. A row is
+    `success: false` with an `error` when the id is unknown, repeated in the request,
+    or outside the caller's scope. Rows that pass those checks are deleted together,
+    in one transaction, so either all of them go or none does.
+
+    Example curl:
+    ```
+    curl --location 'http://0.0.0.0:4000/management/v1/users/bulk_delete' \
+        --header 'Authorization: Bearer sk-1234' \
+        --header 'Content-Type: application/json' \
+        --data '{"user_ids": ["user-1", "user-2"]}'
+    ```
+    """
+    try:
+        from litellm.proxy.proxy_server import (
+            litellm_proxy_admin_name,
+            prisma_client,
+            proxy_logging_obj,
+            user_api_key_cache,
+        )
+
+        if prisma_client is None:
+            raise ManagementProblem(
+                ProblemDetail(
+                    type=f"{PROBLEM_TYPE_BASE}database-not-connected",
+                    title="Database not connected",
+                    status=503,
+                    detail=CommonProxyErrors.db_not_connected_error.value,
+                )
+            )
+
+        results: Final = await bulk_delete_users(
+            data=data,
+            user_api_key_dict=user_api_key_dict,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+            litellm_proxy_admin_name=litellm_proxy_admin_name,
+            litellm_changed_by=litellm_changed_by,
+        )
+        return BulkDeleteUsersResponse(data=results)
+
+    except ManagementProblem:
+        raise
+    except Exception as e:  # noqa: BLE001  # a driver error answers as a problem document, not the OpenAI error shape
+        verbose_proxy_logger.exception(
+            "litellm.proxy.management_endpoints.management_v1.users.bulk_delete_users_action(): Exception occured - %s",
+            e,
+        )
+        raise ManagementProblem(
+            ProblemDetail(
+                type=f"{PROBLEM_TYPE_BASE}internal-server-error",
+                title="Internal server error",
+                status=500,
+                detail="Failed to delete users.",
+            )
+        )

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -15,13 +15,16 @@ import datetime
 import json
 from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
 from json import JSONDecodeError
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast
+from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
 from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
 
+import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.constants import LITELLM_PROXY_ADMIN_NAME
@@ -51,6 +54,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.litellm_license import AUTO_ROUTER_LICENSE_REMEDY
+from litellm.proxy.auth.team_grants import team_model_aliases
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.config_sync_pubsub import (
     coordination_redis_cache,
@@ -65,6 +69,7 @@ from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
 from litellm.proxy.management_endpoints.common_utils import _is_user_team_admin
 from litellm.proxy.management_endpoints.team_endpoints import (
     _refresh_cached_team,
+    append_team_models,
     team_model_add,
     team_model_delete,
 )
@@ -76,6 +81,13 @@ from litellm.proxy.management_helpers.access_group_model_sync import (
     sync_access_groups_for_renamed_model,
 )
 from litellm.proxy.management_helpers.audit_logs import create_object_audit_log
+from litellm.proxy.management_helpers.auto_router_permissions import (
+    MemberAutoRouterWrite,
+    StoredAutoRouterIdentity,
+    authorize_member_auto_router_dependencies,
+    authorize_member_auto_router_team,
+    authorize_member_auto_router_write,
+)
 from litellm.proxy.spend_tracking.ptu_feature_flag import (
     PTU_COST_ATTRIBUTION_ENV_VAR,
     is_ptu_cost_attribution_enabled,
@@ -122,12 +134,14 @@ from litellm.types.router import (
     GenericLiteLLMParams,
     ModelInfo,
     updateDeployment,
+    updateLiteLLMParams,
 )
 from litellm.types.utils import without_server_derived_pricing
 from litellm.utils import get_utc_datetime
 
 if TYPE_CHECKING:
     from prisma import models as prisma_models
+    from prisma import types as prisma_types
 
 router: Final = APIRouter()
 
@@ -180,6 +194,24 @@ class _ProxyModelTable(Protocol):
 class _TxModelTables(Protocol):
     litellm_proxymodeltable: _ProxyModelTable
 
+    async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: ...
+
+
+@runtime_checkable
+class _TransactionFactory(Protocol):
+    def __call__(self, *, timeout: datetime.timedelta = ...) -> AbstractAsyncContextManager[_TxModelTables]: ...
+
+
+class _ModelTransactionClient(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True, from_attributes=True)
+
+    tx: _TransactionFactory
+
+
+@dataclass(frozen=True, slots=True)
+class _TransactionClient:
+    db: _TxModelTables
+
 
 _RowT = TypeVar("_RowT")
 
@@ -213,7 +245,7 @@ def _proxy_model_table(prisma_client: PrismaClient) -> _ProxyModelTable:
 
 
 def _repo_team_table(prisma_client: PrismaClient) -> _TeamLookupTable:
-    return TeamRepository(prisma_client).table
+    return TeamRepository(WriterPinnedClient(prisma_client.db)).table
 
 
 def _db_team_table(prisma_client: PrismaClient) -> _TeamTable:
@@ -353,6 +385,25 @@ def _effective_complexity_router_params(
     )
 
 
+def _member_auto_router_marker_for_update(
+    *,
+    incoming_params: updateLiteLLMParams | None,
+    existing: Deployment,
+    member_write: MemberAutoRouterWrite | None,
+) -> bool | None:
+    if member_write is not None:
+        return True
+    if not existing.model_info.member_auto_router:
+        return None
+    if incoming_params is None:
+        return True
+    if any(getattr(incoming_params, field, None) is not None for field in STRATEGY_ROUTER_PARAM_FIELDS):
+        return False
+    if incoming_params.model is not None and incoming_params.model != _effective_model(None, existing.litellm_params):
+        return False
+    return True
+
+
 def _decrypted_model(stored_model: object) -> str | None:
     if not isinstance(stored_model, str):
         return None
@@ -385,7 +436,11 @@ def _raise_on_tuning_quota_violation(
 
 @asynccontextmanager
 async def _auto_router_capability_slot(
-    prisma_client: PrismaClient, *, effective_params: Mapping[str, object], model_id: str | None
+    prisma_client: PrismaClient,
+    *,
+    effective_params: Mapping[str, object],
+    model_id: str | None,
+    member_write: MemberAutoRouterWrite | None = None,
 ) -> AsyncGenerator[_ProxyModelTable, None]:
     """Hand out the model table to write through while the row's claim on a licensed capability is settled.
 
@@ -394,9 +449,8 @@ async def _auto_router_capability_slot(
     (a statement's snapshot predates anything it locks), so pods cannot both pass the count:
     the DB rows (any pod, either JSON shape) plus this proxy's config.yaml routers are judged
     against the license limit and the write is refused with a 403 before it happens. The row
-    being edited keeps its own slot through ``model_id``. Every other write, and every write on
-    an unlimited license, goes through the repository table with no lock. Only the row write
-    itself may run inside: anything that needs a second connection (the team model bookkeeping)
+    being edited keeps its own slot through ``model_id``. Member writes also recheck their
+    authorization under this lock. Team model bookkeeping needs a second connection and
     must wait until the transaction has committed and the lock is released. The transaction
     writes bypass the repository's publish-on-write, so the config change is published once
     after commit, the way delete_team_models does.
@@ -408,6 +462,7 @@ async def _auto_router_capability_slot(
         _license_check,  # pyright: ignore[reportPrivateUsage]  # existing capability slot reads the proxy license singleton
         heuristic_v1_tuning_baselines,
         llm_router,
+        premium_user,
     )
 
     limit: Final = _license_check.auto_router_capability_limit()
@@ -415,13 +470,96 @@ async def _auto_router_capability_slot(
     baselines: Final = heuristic_v1_tuning_baselines
     tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
     judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
-    if limit is None or (capability is None and not judges_tuning):
+    if member_write is None and (limit is None or (capability is None and not judges_tuning)):
         yield _proxy_model_table(prisma_client)
         return
-    async with prisma_client.db.tx() as tx_ctx:
+    transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
+    transaction: Final = (
+        transaction_client.tx(timeout=datetime.timedelta(seconds=30))
+        if member_write is not None
+        else transaction_client.tx()
+    )
+    async with transaction as tx_ctx:
         tables: Final[_TxModelTables] = tx_ctx
         await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY)
         config_rows: Final = () if llm_router is None else tuple(llm_router.config_deployments())
+        if member_write is not None:
+            if member_write.model_id is not None:
+                await tx_ctx.query_raw(
+                    'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE',
+                    member_write.model_id,
+                )
+            pinned_client: Final = _TransactionClient(tx_ctx)
+            team_where: Final[prisma_types.LiteLLM_TeamTableWhereUniqueInput] = {"team_id": member_write.team_id}
+            team_include: Final[prisma_types.LiteLLM_TeamTableInclude] = {"litellm_model_table": True}
+            team_row: Final = await TeamRepository(pinned_client).table.find_unique(
+                where=team_where, include=team_include
+            )
+            if team_row is None or llm_router is None:
+                raise HTTPException(status_code=403, detail="The auto router's team or model catalog is unavailable.")
+            team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
+            authorize_member_auto_router_team(
+                user_api_key_dict=member_write.actor, team=team, premium_user=premium_user
+            )
+            if member_write.model_id is not None:
+                model_where: Final[prisma_types.LiteLLM_ProxyModelTableWhereInput] = {"model_id": member_write.model_id}
+                current_row: Final = await tables.litellm_proxymodeltable.find_unique(where=model_where)
+                current_identity: Final = (
+                    StoredAutoRouterIdentity.model_validate(current_row.model_dump())
+                    if current_row is not None
+                    else None
+                )
+                current_model: Final = (
+                    Deployment.model_validate(current_row.model_dump()) if current_row is not None else None
+                )
+                if (
+                    current_identity is None
+                    or current_identity.created_by != member_write.actor.user_id
+                    or current_model is None
+                    or current_model.model_info.team_id != member_write.team_id
+                ):
+                    raise HTTPException(status_code=403, detail="Team members can update only their own auto routers.")
+                if current_identity.updated_at != member_write.updated_at:
+                    raise HTTPException(status_code=409, detail="This auto router changed. Reload it before updating.")
+            else:
+                all_models: Final[prisma_types.LiteLLM_ProxyModelTableWhereInput] = {}
+                rows_for_names: Final = await tables.litellm_proxymodeltable.find_many(where=all_models)
+                stored_names: Final = tuple(
+                    (
+                        row.model_name,
+                        model_info_as_mapping(row.model_info),
+                    )
+                    for row in rows_for_names
+                )
+                config_names: Final = tuple(
+                    (str(row.get("model_name", "")), model_info_as_mapping(row.get("model_info")))
+                    for row in config_rows
+                )
+                team_aliases: Final = team_model_aliases(team)
+                aliases: Final = (
+                    *(llm_router.model_group_alias or ()),
+                    *(litellm.model_alias_map or ()),
+                    *(team_aliases or ()),
+                )
+                if member_write.public_name in aliases or any(
+                    fnmatchcase(
+                        member_write.public_name,
+                        str(info.get("team_public_model_name") or name)
+                        if info is not None and info.get("team_id") == member_write.team_id
+                        else name,
+                    )
+                    for name, info in (*stored_names, *config_names)
+                    if info is None or info.get("team_id") in (None, member_write.team_id)
+                ):
+                    raise HTTPException(status_code=409, detail="This auto-router name is already used by a model.")
+            await authorize_member_auto_router_dependencies(
+                config=member_write.config,
+                default_model=member_write.default_model,
+                user_api_key_dict=member_write.actor,
+                team=team,
+                prisma_client=pinned_client,
+                llm_router=llm_router,
+            )
         if capability is not None:
             rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw(
                 _CAPABILITY_DB_ROWS_SQL[capability.key], model_id or ""
@@ -434,7 +572,7 @@ async def _auto_router_capability_slot(
                     status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
                 )
         if judges_tuning and baselines is not None:
-            model_rows: Final = await ModelRepository(WriterPinnedClient(tx_ctx)).find_all_except(model_id or "")
+            model_rows: Final = await ModelRepository(_TransactionClient(tx_ctx)).find_all_except(model_id or "")
             _raise_on_tuning_quota_violation(
                 candidate=tuning_candidate,
                 others=tuple(
@@ -883,11 +1021,39 @@ async def patch_model(
                 param=None,
             )
 
-        await ModelManagementAuthChecks.can_user_make_model_call(
+        write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
             model_params=db_model,
             user_api_key_dict=user_api_key_dict,
             prisma_client=prisma_client,
             premium_user=premium_user,
+            member_operation="update",
+            incoming_model_params=patch_data,
+        )
+        member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
+        member_marker: Final = _member_auto_router_marker_for_update(
+            incoming_params=patch_data.litellm_params, existing=db_model, member_write=member_write
+        )
+        marker_info: Final = (
+            ModelInfo(id=db_model.model_info.id)
+            if member_write is not None
+            else patch_data.model_info or ModelInfo(id=db_model.model_info.id)
+        )
+        effective_info: Final = (
+            marker_info.model_copy(update=MappingProxyType({"member_auto_router": member_marker}))
+            if member_marker is not None
+            else patch_data.model_info
+        )
+        effective_patch: Final = (
+            patch_data.model_copy(
+                update=MappingProxyType(
+                    {
+                        "model_name": None if member_write is not None else patch_data.model_name,
+                        "model_info": effective_info,
+                    }
+                )
+            )
+            if member_marker is not None
+            else patch_data
         )
 
         # Pause/resume (`blocked`) is a proxy-admin-only privilege. Team admins
@@ -933,13 +1099,14 @@ async def patch_model(
                 prisma_client,
                 effective_params=effective_params,
                 model_id=model_id,
+                member_write=member_write,
             ) as table:
                 return await table.update(where={"model_id": model_id}, data=update_data)
 
         # Handle team model updates with proper alias management
         updated_model: Final = await _update_team_model_in_db(
             db_model=db_model,
-            patch_data=patch_data,
+            patch_data=effective_patch,
             user_api_key_dict=user_api_key_dict,
             prisma_client=prisma_client,
             write_row=write_row,
@@ -1218,7 +1385,7 @@ async def _add_team_model_to_db(
     user_api_key_dict: UserAPIKeyAuth,
     prisma_client: PrismaClient,
     slot: AbstractAsyncContextManager[_ProxyModelTable] | None = None,
-) -> "_ProxyModelRow | LiteLLM_ProxyModelTable":
+) -> "_ProxyModelRow | LiteLLM_ProxyModelTable | None":
     """
     If 'team_id' is provided,
 
@@ -1226,6 +1393,8 @@ async def _add_team_model_to_db(
     - store the model in the db with the unique 'model_name'
     - add the public model name to the team's allowed models list
     """
+    from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
+
     _team_id: Final = model_params.model_info.team_id
     if _team_id is None:
         return None
@@ -1253,13 +1422,14 @@ async def _add_team_model_to_db(
     )
 
     if original_model_name:
-        await team_model_add(
+        await append_team_models(
             data=TeamModelAddRequest(
                 team_id=_team_id,
                 models=[original_model_name],
             ),
-            http_request=Request(scope={"type": "http"}),
-            user_api_key_dict=user_api_key_dict,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
         )
 
     return model_response
@@ -1787,9 +1957,16 @@ class ModelManagementAuthChecks:
         prisma_client: PrismaClient,
         premium_user: bool,
         allow_missing_team: bool = False,
-    ) -> Literal[True]:
+        member_operation: Literal["create", "update"] | None = None,
+        incoming_model_params: updateDeployment | None = None,
+    ) -> Literal[True] | MemberAutoRouterWrite:
+        if user_api_key_dict.user_role in (
+            LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+            LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
+        ):
+            raise HTTPException(status_code=403, detail="View-only users cannot manage models.")
         ## Check team model auth
-        if model_params.model_info is not None and model_params.model_info.team_id is not None:
+        if model_params.model_info.team_id is not None:
             team_obj_row: Final = await _repo_team_table(prisma_client).find_unique(
                 where={"team_id": model_params.model_info.team_id}
             )
@@ -1810,6 +1987,27 @@ class ModelManagementAuthChecks:
                 )
             team_obj: Final = LiteLLM_TeamTable.model_validate(team_obj_row.model_dump())
 
+            if (
+                member_operation is not None
+                and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
+                and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj)
+            ):
+                from litellm.proxy.proxy_server import llm_router
+
+                if llm_router is None or (member_operation == "update" and incoming_model_params is None):
+                    raise HTTPException(
+                        status_code=400, detail="An auto-router configuration and model catalog are required."
+                    )
+                return await authorize_member_auto_router_write(
+                    incoming=incoming_model_params if incoming_model_params is not None else model_params,
+                    existing=model_params if member_operation == "update" else None,
+                    user_api_key_dict=user_api_key_dict,
+                    team=team_obj,
+                    premium_user=premium_user,
+                    prisma_client=prisma_client,
+                    llm_router=llm_router,
+                )
+
             return ModelManagementAuthChecks.can_user_make_team_model_call(
                 team_id=model_params.model_info.team_id,
                 user_api_key_dict=user_api_key_dict,
@@ -2067,12 +2265,14 @@ async def add_new_model(
             )
 
         ## Auth check
-        await ModelManagementAuthChecks.can_user_make_model_call(
+        write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
             model_params=model_params,
             user_api_key_dict=user_api_key_dict,
             prisma_client=prisma_client,
             premium_user=premium_user,
+            member_operation="create",
         )
+        member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
 
         ModelManagementAuthChecks.can_user_attach_credential(
             litellm_params=model_params.litellm_params,
@@ -2094,9 +2294,14 @@ async def add_new_model(
             enforced=bool(general_settings.get(ENFORCE_RPM_TPM_ON_MODEL_ADD_SETTING, False)),
         )
 
-        model_params.model_info = ModelInfo(  # rebind-ok: downstream team-model handling mutates this same object
+        clean_model_info: Final = ModelInfo(
             **without_server_derived_pricing(model_params.model_info.model_dump(exclude_none=True))
         )
+        model_params.model_info = (  # rebind-ok: downstream team-model handling mutates this same object
+            clean_model_info.model_copy(update=MappingProxyType({"member_auto_router": True}))
+            if member_write is not None
+            else clean_model_info
+        )
 
         model_response: prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None = None
         # update DB
@@ -2129,6 +2334,7 @@ async def add_new_model(
                             None,
                         ),
                         model_id=priced_model_params.model_info.id,
+                        member_write=member_write,
                     ),
                 )
                 reload_outcome = await proxy_config.add_deployment(
@@ -2259,12 +2465,15 @@ async def update_model(
                 raise Exception("model not found")
         deployment: Final = Deployment(**_existing_litellm_params.model_dump())
 
-        await ModelManagementAuthChecks.can_user_make_model_call(
+        write_authorization: Final = await ModelManagementAuthChecks.can_user_make_model_call(
             model_params=deployment,
             user_api_key_dict=user_api_key_dict,
             prisma_client=prisma_client,
             premium_user=premium_user,
+            member_operation="update",
+            incoming_model_params=model_params,
         )
+        member_write: Final = write_authorization if isinstance(write_authorization, MemberAutoRouterWrite) else None
 
         ModelManagementAuthChecks.can_user_attach_credential(
             litellm_params=model_params.litellm_params,
@@ -2285,6 +2494,9 @@ async def update_model(
         effective_params: Final = _effective_complexity_router_params(
             model_params.litellm_params, deployment.litellm_params
         )
+        member_marker: Final = _member_auto_router_marker_for_update(
+            incoming_params=model_params.litellm_params, existing=deployment, member_write=member_write
+        )
 
         # update DB
         if store_model_in_db is True:
@@ -2317,15 +2529,30 @@ async def update_model(
                 and deployment.model_info.team_id is None
                 else None
             )
-            _data: Final[dict[str, str]] = {
+            base_update: Final[PrismaCompatibleUpdateDBModel] = {
                 "litellm_params": json.dumps(merged_dictionary),
                 "updated_by": user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
-                **({} if renamed_to is None else {"model_name": renamed_to}),
             }
+            renamed_update: Final[PrismaCompatibleUpdateDBModel] = (
+                {**base_update, "model_name": renamed_to}  # mutable-ok: Prisma serializes only concrete update dicts
+                if renamed_to is not None
+                else base_update
+            )
+            _data: Final[PrismaCompatibleUpdateDBModel] = (
+                {  # mutable-ok: Prisma serializes only concrete update dicts
+                    **renamed_update,
+                    "model_info": deployment.model_info.model_copy(
+                        update=MappingProxyType({"member_auto_router": member_marker})
+                    ).model_dump_json(exclude_none=True),
+                }
+                if member_marker is not None
+                else renamed_update
+            )
             async with _auto_router_capability_slot(
                 prisma_client,
                 effective_params=effective_params,
                 model_id=_model_id,
+                member_write=member_write,
             ) as table:
                 model_response: Final = await table.update(
                     where={"model_id": _model_id},
@@ -2421,7 +2648,6 @@ async def update_public_model_groups(
     """
     try:
         # Update the public model groups
-        import litellm
         from litellm.proxy.proxy_server import proxy_config, store_model_in_db
 
         # Check if user has admin permissions
@@ -2496,7 +2722,6 @@ async def update_useful_links(
     """
     try:
         # Update the public model groups
-        import litellm
         from litellm.proxy.proxy_server import proxy_config
 
         # Check if user has admin permissions

litellm/proxy/management_endpoints/router_weights.py

@@ -0,0 +1,129 @@
+from abc import abstractmethod
+from collections.abc import Mapping
+from typing import Annotated, Final, Protocol
+
+from fastapi import HTTPException
+from pydantic import BaseModel, BeforeValidator, ValidationError
+
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.types.router_weights import RouterWeights
+
+
+class _StoredModel(Protocol):
+    @property
+    @abstractmethod
+    def model_id(self) -> str:
+        pass
+
+
+class _ModelDb(Protocol):
+    @property
+    @abstractmethod
+    def litellm_proxymodeltable(self) -> TableActions[_StoredModel]:
+        pass
+
+
+class _PrismaClient(Protocol):
+    @property
+    @abstractmethod
+    def db(self) -> _ModelDb:
+        pass
+
+
+class _Router(Protocol):
+    @abstractmethod
+    def get_deployment(self, model_id: str) -> object | None:
+        pass
+
+
+class _RouterWeightSettings(BaseModel):
+    weights: RouterWeights | None = None
+
+
+class _RouterWeightModelInfo(BaseModel):
+    team_id: str | None = None
+    db_model: bool | None = None
+    team_public_model_name: str | None = None
+
+
+def _router_weight_model_info(value: object) -> _RouterWeightModelInfo:
+    if isinstance(value, str):
+        return _RouterWeightModelInfo.model_validate_json(value)
+    return _RouterWeightModelInfo.model_validate(value or {}, from_attributes=True)
+
+
+class _RouterWeightDeployment(BaseModel):
+    model_name: str
+    model_info: Annotated[_RouterWeightModelInfo, BeforeValidator(_router_weight_model_info)]
+
+
+def _validate_router_weight_reference(
+    model_group: str,
+    deployment_id: str,
+    team_id: str | None,
+    stored: _RouterWeightDeployment | None,
+    configured: object | None,
+) -> None:
+    reference: Final = (
+        stored
+        if stored is not None
+        else (
+            _RouterWeightDeployment.model_validate(configured, from_attributes=True) if configured is not None else None
+        )
+    )
+    if (
+        reference is None
+        or (stored is None and reference.model_info.db_model)
+        or (reference.model_info.team_id is not None and reference.model_info.team_id != team_id)
+    ):
+        raise HTTPException(status_code=400, detail=f"Unknown deployment ID in router weights: {deployment_id}")
+    canonical_group: Final = (
+        reference.model_info.team_public_model_name if reference.model_info.team_id is not None else None
+    ) or reference.model_name
+    if model_group != canonical_group:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Deployment {deployment_id} does not belong to model group {model_group}",
+        )
+
+
+async def validate_router_settings_weights(
+    router_settings: BaseModel | Mapping[str, object] | None,
+    *,
+    team_id: str | None,
+    prisma_client: _PrismaClient | None,
+    llm_router: _Router | None,
+) -> None:
+    try:
+        weights: Final = (
+            _RouterWeightSettings.model_validate(router_settings, from_attributes=True).weights
+            if router_settings is not None
+            else None
+        )
+    except ValidationError:
+        raise HTTPException(
+            status_code=400,
+            detail="Invalid router weights. Replace or clear router_settings.weights.",
+        ) from None
+    if not weights:
+        return
+    deployment_ids: Final = frozenset(deployment_id for group in weights.values() for deployment_id in group)
+    if not deployment_ids:
+        return
+    if prisma_client is None:
+        raise HTTPException(status_code=503, detail="Database unavailable while validating router weights")
+    stored_models: Final = await prisma_client.db.litellm_proxymodeltable.find_many(
+        where={"model_id": {"in": list(deployment_ids)}}
+    )
+    stored_by_id: Final = {
+        row.model_id: _RouterWeightDeployment.model_validate(row, from_attributes=True) for row in stored_models
+    }
+    for model_group, group_weights in weights.items():
+        for deployment_id in group_weights:
+            _validate_router_weight_reference(
+                model_group,
+                deployment_id,
+                team_id,
+                stored_by_id.get(deployment_id),
+                llm_router.get_deployment(model_id=deployment_id) if llm_router is not None else None,
+            )

litellm/proxy/management_endpoints/team_endpoints.py

@@ -112,6 +112,7 @@ from litellm.proxy.management_endpoints.common_utils import (
 from litellm.proxy.management_endpoints.organization_endpoints import (
     add_member_to_organization,
 )
+from litellm.proxy.management_endpoints.router_weights import validate_router_settings_weights
 from litellm.proxy.management_endpoints.tag_management_endpoints import (
     get_daily_activity,
 )
@@ -1288,6 +1289,7 @@ async def new_team(
             create_audit_log_for_update,
             general_settings,
             litellm_proxy_admin_name,
+            llm_router,
             prisma_client,
             user_api_key_cache,
         )
@@ -1462,6 +1464,13 @@ async def new_team(
             user_api_key_dict=user_api_key_dict,
         )
 
+        await validate_router_settings_weights(
+            data.router_settings,
+            team_id=data.team_id,
+            prisma_client=prisma_client,
+            llm_router=llm_router,
+        )
+
         ## ADD TO MODEL TABLE
         _model_id = None
         if data.model_aliases is not None and isinstance(data.model_aliases, dict):
@@ -2075,6 +2084,13 @@ async def update_team(
             user_api_key_dict=user_api_key_dict,
         )
 
+        await validate_router_settings_weights(
+            data.router_settings,
+            team_id=data.team_id,
+            prisma_client=prisma_client,
+            llm_router=llm_router,
+        )
+
         _existing_team_metadata: Final[object] = getattr(existing_team_row, "metadata", None)
         enforce_output_token_estimates_are_admin_only(
             data=data,
@@ -3309,7 +3325,8 @@ async def team_member_delete(
     }'
     ```
     """
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
+    from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache
 
     if prisma_client is None:
         raise HTTPException(status_code=500, detail={"error": "No db connected"})
@@ -3447,6 +3464,25 @@ async def team_member_delete(
                 }
             )
 
+    await delete_cache_team_object(
+        team_id=data.team_id,
+        team_alias=existing_team_row.team_alias,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    await delete_cache_key_objects(
+        hashed_tokens=tuple(key.token for key in keys_to_delete),
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    await evict_and_broadcast(cache_keys=tuple(sorted(user_ids_to_delete)), user_api_key_cache=user_api_key_cache)
+    for user_id in sorted(user_ids_to_delete):
+        await invalidate_team_member_spend_state(
+            user_id=user_id,
+            team_id=data.team_id,
+            user_api_key_cache=user_api_key_cache,
+        )
+
     _emit_team_members_metric(existing_team_row)
 
     return existing_team_row
@@ -5668,6 +5704,21 @@ async def team_model_add(
             detail={"error": "Only proxy admin or team admin can modify team models"},
         )
 
+    return await append_team_models(
+        data=data,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+
+
+async def append_team_models(
+    *,
+    data: TeamModelAddRequest,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging,
+) -> "prisma_models.LiteLLM_TeamTable":
     # Atomic array append with dedup at the database level so concurrent
     # BYOK model creates don't overwrite each other's team.models entries.
     # When the team currently has models=[] (unrestricted access), the

litellm/proxy/management_endpoints/ui_sso.py

@@ -3592,6 +3592,7 @@ class SSOAuthenticationHandler:
         verbose_proxy_logger.info("user_defined_values for creating ui key: %s", user_defined_values)
 
         response: Final = await generate_key_helper_fn(
+            llm_router=None,
             request_type="key",
             duration=LITELLM_UI_SESSION_DURATION,
             key_max_budget=litellm.max_ui_session_budget,

litellm/proxy/management_helpers/auto_router_permissions.py

@@ -0,0 +1,345 @@
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import datetime
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
+
+from fastapi import HTTPException
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.models.organization import LiteLLM_OrganizationTable
+from litellm.models.project import LiteLLM_ProjectTable
+from litellm.proxy._types import (
+    UI_TEAM_ID,
+    CommonProxyErrors,
+    KeyManagementRoutes,
+    LiteLLM_TeamMembership,
+    LiteLLM_TeamTable,
+    LitellmUserRoles,
+    UserAPIKeyAuth,
+)
+from litellm.proxy.auth.auth_checks import (
+    _check_team_member_model_access,  # pyright: ignore[reportPrivateUsage]  # shared membership authorization owner
+    can_key_call_model,
+    can_org_access_model,
+    can_project_access_model,
+    can_team_access_model,
+)
+from litellm.proxy.auth.team_grants import team_model_aliases
+from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
+from litellm.repositories.organization_repository import OrganizationRepository
+from litellm.repositories.prisma_protocols import DatabaseClient
+from litellm.repositories.project_repository import ProjectRepository
+from litellm.repositories.table_repositories import TeamMembershipRepository
+from litellm.router import Router
+from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model, strategy_router_dependencies
+from litellm.types.management_endpoints.auto_router_endpoints import RequestComplexityRouterConfig
+from litellm.types.router import Deployment, updateDeployment
+
+if TYPE_CHECKING:
+    from prisma import types as prisma_types
+
+
+class _MemberRouterThinking(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    type: Literal["enabled", "disabled", "adaptive"]
+    budget_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
+
+
+class _MemberRouterGenerationParams(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    reasoning_effort: str | None = None
+    thinking: _MemberRouterThinking | None = None
+    verbosity: Literal["low", "medium", "high"] | None = None
+    max_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
+    max_completion_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
+    max_output_tokens: int | None = Field(default=None, gt=0, le=1_000_000)
+    temperature: float | None = Field(default=None, ge=0, le=2, allow_inf_nan=False)
+    top_p: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False)
+    frequency_penalty: float | None = Field(default=None, ge=-2, le=2, allow_inf_nan=False)
+    presence_penalty: float | None = Field(default=None, ge=-2, le=2, allow_inf_nan=False)
+    seed: int | None = None
+    stop: str | tuple[str, ...] | None = None
+
+
+class _MemberComplexityRouterConfig(RequestComplexityRouterConfig):
+    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
+
+
+class _RouterConfigSource(BaseModel):
+    model: str | None = None
+    complexity_router_config: Mapping[str, object] | None = None
+
+
+class _MembershipKey(TypedDict):
+    user_id: ReadOnly[str]
+    team_id: ReadOnly[str]
+
+
+class _MembershipWhere(TypedDict):
+    user_id_team_id: ReadOnly[_MembershipKey]
+
+
+@dataclass(frozen=True, slots=True)
+class MemberAutoRouterDependencyObjects:
+    membership: LiteLLM_TeamMembership | None
+    organization: LiteLLM_OrganizationTable | None
+    project: LiteLLM_ProjectTable | None
+
+
+def authorize_member_auto_router_team(
+    *, user_api_key_dict: UserAPIKeyAuth, team: LiteLLM_TeamTable, premium_user: bool
+) -> None:
+    if not premium_user:
+        raise HTTPException(status_code=403, detail=CommonProxyErrors.not_premium_user.value)
+    if (
+        user_api_key_dict.user_role
+        not in (LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.TEAM, LitellmUserRoles.ORG_ADMIN)
+        or not user_api_key_dict.user_id
+        or not any(member.user_id == user_api_key_dict.user_id for member in team.members_with_roles)
+        or user_api_key_dict.team_id not in (None, UI_TEAM_ID, team.team_id)
+        or team.blocked
+        or KeyManagementRoutes.AUTO_ROUTER_MANAGE.value not in (team.team_member_permissions or ())
+    ):
+        raise HTTPException(status_code=403, detail="This team does not allow you to manage your own auto routers.")
+
+
+def validate_member_auto_router_config(config: Mapping[str, object]) -> RequestComplexityRouterConfig:
+    try:
+        validated: Final = _MemberComplexityRouterConfig.model_validate(config)
+        for entries in validated.tier_model_configs.values():
+            for entry in entries:
+                _MemberRouterGenerationParams.model_validate(entry.litellm_params)
+        return validated
+    except ValidationError as exc:
+        location: Final = ".".join(str(part) for part in exc.errors()[0]["loc"])
+        raise HTTPException(status_code=400, detail=f"Invalid member auto-router configuration at {location}.") from exc
+
+
+async def authorize_member_auto_router_dependencies(
+    *,
+    config: RequestComplexityRouterConfig,
+    default_model: str | None,
+    user_api_key_dict: UserAPIKeyAuth,
+    team: LiteLLM_TeamTable,
+    prisma_client: DatabaseClient | None,
+    llm_router: Router,
+    dependency_objects: MemberAutoRouterDependencyObjects | None = None,
+) -> None:
+    from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
+
+    if team.blocked:
+        raise HTTPException(status_code=403, detail="This auto router's team is blocked.")
+    aliases: Final = team_model_aliases(team)
+    alias_dict: Final = (
+        dict(aliases) if aliases is not None else None  # mutable-ok: auth model and helpers require dict
+    )
+    scoped_actor: Final = user_api_key_dict.model_copy(
+        update=MappingProxyType({"team_id": team.team_id, "team_models": team.models, "team_model_aliases": alias_dict})
+    )
+    objects: Final = (
+        dependency_objects
+        if dependency_objects is not None
+        else await _load_member_auto_router_dependency_objects(
+            user_api_key_dict=scoped_actor, team=team, prisma_client=prisma_client
+        )
+    )
+    if team.organization_id and objects.organization is None:
+        raise HTTPException(status_code=403, detail="The auto router's organization is unavailable.")
+    if scoped_actor.project_id and (
+        objects.project is None or objects.project.team_id != team.team_id or objects.project.blocked
+    ):
+        raise HTTPException(status_code=403, detail="The auto router's project is unavailable.")
+    dependencies: Final = strategy_router_dependencies(
+        MappingProxyType(
+            {
+                "model": "auto_router/complexity_router",
+                "complexity_router_config": config.model_dump(exclude_none=True),
+                "complexity_router_default_model": default_model,
+            }
+        )
+    )
+    for model, deployments in (
+        (dependency.model_name, llm_router.get_model_list(model_name=dependency.model_name, team_id=team.team_id))
+        for dependency in dependencies
+    ):
+        if not deployments or any(
+            classify_strategy_router_model(_RouterConfigSource.model_validate(deployment["litellm_params"]).model or "")
+            is not None
+            for deployment in deployments
+        ):
+            raise HTTPException(status_code=400, detail=f"Auto-router target {model!r} must be a configured model.")
+        await can_team_access_model(
+            model=model,
+            team_object=team,
+            llm_router=llm_router,
+            team_model_aliases=alias_dict,
+            prisma_client=prisma_client,
+        )
+        await can_key_call_model(
+            model=model,
+            llm_model_list=None,
+            valid_token=scoped_actor,
+            llm_router=llm_router,
+            prisma_client=prisma_client,
+        )
+        await _check_team_member_model_access(
+            model=model,
+            team_object=team,
+            valid_token=scoped_actor,
+            llm_router=llm_router,
+            prisma_client=None,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+            team_membership=objects.membership,
+            team_membership_loaded=True,
+        )
+        if objects.organization is not None:
+            can_org_access_model(model=model, org_object=objects.organization, llm_router=llm_router)
+        if objects.project is not None:
+            can_project_access_model(model=model, project_object=objects.project, llm_router=llm_router)
+
+
+async def _load_member_auto_router_dependency_objects(
+    *, user_api_key_dict: UserAPIKeyAuth, team: LiteLLM_TeamTable, prisma_client: DatabaseClient | None
+) -> MemberAutoRouterDependencyObjects:
+    if prisma_client is None:
+        raise HTTPException(status_code=503, detail="Cannot verify auto-router model access without a database")
+    membership_where: Final[_MembershipWhere] = {
+        "user_id_team_id": {"user_id": user_api_key_dict.user_id or "", "team_id": team.team_id}
+    }
+    membership_include: Final[prisma_types.LiteLLM_TeamMembershipInclude] = {"litellm_budget_table": True}
+    membership_row: Final = (
+        await TeamMembershipRepository(prisma_client).table.find_unique(
+            where=membership_where, include=membership_include
+        )
+        if user_api_key_dict.user_id
+        else None
+    )
+    membership: Final = (
+        LiteLLM_TeamMembership.model_validate(membership_row.model_dump()) if membership_row is not None else None
+    )
+    organization: Final = (
+        await OrganizationRepository(prisma_client).find_by_id(team.organization_id) if team.organization_id else None
+    )
+    if team.organization_id and organization is None:
+        raise HTTPException(status_code=403, detail="The auto router's organization is unavailable.")
+    project: Final = (
+        await ProjectRepository(prisma_client).find_by_id(user_api_key_dict.project_id)
+        if user_api_key_dict.project_id
+        else None
+    )
+    return MemberAutoRouterDependencyObjects(membership=membership, organization=organization, project=project)
+
+
+class StoredAutoRouterIdentity(BaseModel):
+    created_by: str | None = None
+    updated_at: datetime | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class MemberAutoRouterWrite:
+    actor: UserAPIKeyAuth
+    team_id: str
+    model_id: str | None
+    public_name: str
+    updated_at: datetime | None
+    config: RequestComplexityRouterConfig
+    default_model: str | None
+
+
+async def authorize_member_auto_router_write(
+    *,
+    incoming: Deployment | updateDeployment,
+    existing: Deployment | None,
+    user_api_key_dict: UserAPIKeyAuth,
+    team: LiteLLM_TeamTable,
+    premium_user: bool,
+    prisma_client: DatabaseClient,
+    llm_router: Router,
+) -> MemberAutoRouterWrite:
+    authorize_member_auto_router_team(user_api_key_dict=user_api_key_dict, team=team, premium_user=premium_user)
+    stored: Final = StoredAutoRouterIdentity.model_validate(existing.model_dump()) if existing is not None else None
+    if stored is not None and stored.created_by != user_api_key_dict.user_id:
+        raise HTTPException(status_code=403, detail="Team members can update only their own auto routers.")
+    params: Final = incoming.litellm_params
+    if params is None or incoming.model_fields_set - frozenset({"model_name", "litellm_params", "model_info"}):
+        raise HTTPException(status_code=403, detail="Team members may change only auto-router configuration.")
+    if params.model_fields_set - frozenset({"model", "complexity_router_config", "complexity_router_default_model"}):
+        raise HTTPException(status_code=403, detail="Team members may change only auto-router configuration.")
+    info: Final = incoming.model_info
+    if info is not None and (
+        info.model_fields_set - frozenset({"id", "team_id"})
+        or info.team_id not in (None, team.team_id)
+        or (existing is not None and "id" in info.model_fields_set and info.id != existing.model_info.id)
+    ):
+        raise HTTPException(
+            status_code=403, detail="Team members cannot change model ownership or administrative settings."
+        )
+    existing_model: Final = (
+        decrypt_value_helper(existing.litellm_params.model, key="model", return_original_value=True)
+        if existing is not None
+        else None
+    )
+    effective_model: Final = params.model or existing_model
+    if (
+        not isinstance(effective_model, str)
+        or classify_strategy_router_model(effective_model) != "complexity"
+        or (existing is not None and effective_model != existing_model)
+    ):
+        raise HTTPException(status_code=403, detail="Team members may manage only complexity auto routers.")
+    public_name: Final = (
+        existing.model_info.team_public_model_name or existing.model_name
+        if existing is not None
+        else incoming.model_name
+    )
+    if (
+        not public_name
+        or public_name != public_name.strip()
+        or any(character in public_name for character in "*?[]")
+        or public_name.startswith("model_name_")
+    ):
+        raise HTTPException(
+            status_code=400, detail="Choose a non-empty auto-router name without wildcards or internal prefixes."
+        )
+    if existing is not None and incoming.model_name not in (None, public_name, existing.model_name):
+        raise HTTPException(status_code=403, detail="Team members cannot rename an auto router.")
+    supplied_config: Final = _RouterConfigSource.model_validate(params.model_dump()).complexity_router_config
+    raw_config: Final = (
+        supplied_config
+        if supplied_config is not None
+        else _RouterConfigSource.model_validate(existing.litellm_params.model_dump()).complexity_router_config
+        if existing is not None
+        else None
+    )
+    if raw_config is None:
+        raise HTTPException(status_code=400, detail="A complexity_router_config is required.")
+    config: Final = validate_member_auto_router_config(raw_config)
+    stored_default: Final = existing.litellm_params.complexity_router_default_model if existing is not None else None
+    default_model: Final = (
+        params.complexity_router_default_model
+        if params.complexity_router_default_model is not None
+        else decrypt_value_helper(stored_default, key="complexity_router_default_model", return_original_value=True)
+        if stored_default is not None
+        else None
+    )
+    await authorize_member_auto_router_dependencies(
+        config=config,
+        default_model=default_model,
+        user_api_key_dict=user_api_key_dict,
+        team=team,
+        prisma_client=prisma_client,
+        llm_router=llm_router,
+    )
+    return MemberAutoRouterWrite(
+        actor=user_api_key_dict,
+        team_id=team.team_id,
+        model_id=existing.model_info.id if existing is not None else None,
+        public_name=public_name,
+        updated_at=stored.updated_at if stored is not None else None,
+        config=config,
+        default_model=default_model,
+    )

litellm/proxy/management_helpers/bulk_user_creation.py

@@ -0,0 +1,871 @@
+"""Batched internal user creation behind `POST /management/v1/users/bulk`.
+
+The batch is validated with set queries, user rows land in one `create_many`, and every
+referenced team is written once under its advisory lock instead of once per user.
+"""
+
+import asyncio
+import json
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar
+
+from fastapi import HTTPException, Request
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm._logging import verbose_proxy_logger
+from litellm._uuid import uuid
+from litellm.integrations.prometheus import PrometheusLogger
+from litellm.proxy._types import (
+    LiteLLM_TeamTable,
+    LitellmUserRoles,
+    Member,
+    NewUserRequestTeam,
+    OrganizationMemberAddRequest,
+    OrgMember,
+    UserAPIKeyAuth,
+)
+from litellm.proxy.auth.auth_checks import invalidate_team_member_spend_state
+from litellm.proxy.auth.litellm_license import LicenseCheck
+from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
+from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
+from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks
+from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem
+from litellm.proxy.management_endpoints.common_utils import (
+    _is_user_org_admin_for_team,  # pyright: ignore[reportPrivateUsage]  # same team-admin check /user/new uses
+    _is_user_team_admin,  # pyright: ignore[reportPrivateUsage]  # same team-admin check /user/new uses
+    validate_budget_duration,
+)
+from litellm.proxy.management_endpoints.internal_user_endpoints import (
+    _update_internal_new_user_params,  # pyright: ignore[reportPrivateUsage, reportUnknownVariableType]  # /user/new defaults; result validated below
+    check_if_default_team_set,
+)
+from litellm.proxy.management_endpoints.key_management_endpoints import (
+    _check_permissions_caller_permission,  # pyright: ignore[reportPrivateUsage]  # same permission check /user/new uses
+    generate_key_helper_fn,  # pyright: ignore[reportUnknownVariableType]  # legacy untyped helper; result validated by _KEY_RESPONSE
+    metadata_json_with_limits,
+)
+from litellm.proxy.management_endpoints.organization_endpoints import organization_member_add
+from litellm.proxy.management_helpers.access_group_team_sync import TEAM_ADVISORY_LOCK_SQL
+from litellm.proxy.management_helpers.object_permission_utils import (
+    _set_object_permission,  # pyright: ignore[reportPrivateUsage, reportUnknownVariableType]  # shared with /user/new; result validated below
+)
+from litellm.proxy.management_helpers.utils import (
+    _resolve_member_budget_id,  # pyright: ignore[reportPrivateUsage]  # shared with /team/member_add
+)
+from litellm.proxy.utils import PrismaClient
+from litellm.repositories.prisma_protocols import TableActions
+from litellm.repositories.team_repository import TeamRepository
+from litellm.repositories.user_repository import UserRepository
+from litellm.types.proxy.management_endpoints.internal_user_endpoints import (
+    BulkNewUserItem,
+    BulkNewUserMeta,
+    BulkNewUserResponse,
+    UserCreateResult,
+)
+from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail
+
+if TYPE_CHECKING:
+    from prisma import Prisma
+    from prisma import models as prisma_models
+
+    from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+
+BULK_NEW_USER_CONCURRENCY: Final = 10
+
+TeamRole: TypeAlias = Literal["user", "admin"]
+KeyGenerator: TypeAlias = Callable[..., Awaitable[object]]
+_T: Final = TypeVar("_T")
+
+
+@dataclass(frozen=True, slots=True)
+class _RowFailure:
+    index: int
+    user_id: str | None
+    user_email: str | None
+    error: str
+
+
+@dataclass(frozen=True, slots=True)
+class _PendingUser:
+    index: int
+    request: BulkNewUserItem
+    user_id: str
+    teams: tuple[NewUserRequestTeam, ...]
+
+
+class _UserRow(BaseModel):
+    """The `/user/new` body after defaults and object permission were applied."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    user_id: str
+    user_email: str | None = None
+    user_alias: str | None = None
+    user_role: str | None = None
+    team_id: str | None = None
+    max_budget: float | None = None
+    spend: float | None = 0.0
+    models: tuple[str, ...] | None = None
+    metadata: Mapping[str, object] | None = None
+    max_parallel_requests: int | None = None
+    tpm_limit: int | None = None
+    rpm_limit: int | None = None
+    budget_duration: str | None = None
+    allowed_cache_controls: tuple[str, ...] | None = None
+    sso_user_id: str | None = None
+    object_permission_id: str | None = None
+    model_max_budget: Mapping[str, object] | None = None
+    model_rpm_limit: Mapping[str, object] | None = None
+    model_tpm_limit: Mapping[str, object] | None = None
+    mcp_rpm_limit: Mapping[str, int] | None = None
+    tag_rpm_limit: Mapping[str, int] | None = None
+    guardrails: tuple[str, ...] | None = None
+    policies: tuple[str, ...] | None = None
+    prompts: tuple[str, ...] | None = None
+    duration: str | None = None
+    key_alias: str | None = None
+    aliases: Mapping[str, object] | None = None
+    config: Mapping[str, object] | None = None
+    permissions: Mapping[str, object] | None = None
+    blocked: bool | None = None
+    agent_id: str | None = None
+    budget_fallbacks: Mapping[str, tuple[str, ...]] | None = None
+    budget_limits: tuple[Mapping[str, object], ...] | None = None
+    organizations: tuple[str, ...] | None = None
+
+
+_USER_ROW: Final = TypeAdapter(_UserRow)
+
+
+@dataclass(frozen=True, slots=True)
+class _PreparedUser:
+    pending: _PendingUser
+    row: _UserRow
+
+
+@dataclass(frozen=True, slots=True)
+class _TeamAssignment:
+    user_id: str
+    user_email: str | None
+    role: TeamRole
+    max_budget_in_team: float | None
+
+
+@dataclass(frozen=True, slots=True)
+class _TeamWrite:
+    """Outcome of one locked roster write. `failed` maps user ids to the reason they were not added."""
+
+    team_id: str
+    after: tuple[Member, ...]
+    added: frozenset[str]
+    failed: Mapping[str, str]
+
+
+@dataclass(frozen=True, slots=True)
+class _CreatedUser:
+    prepared: _PreparedUser
+    teams: tuple[str, ...]
+    key: str | None
+    errors: tuple[str, ...]
+
+
+_ERROR_DETAIL: Final = TypeAdapter(Mapping[str, object])
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
+
+
+class _KeyResponse(BaseModel):
+    token: str
+
+
+_KEY_RESPONSE: Final = TypeAdapter(_KeyResponse)
+
+
+def _error_message(exc: BaseException) -> str:
+    if not isinstance(exc, HTTPException):
+        return str(exc)
+    try:
+        detail: Final = _ERROR_DETAIL.validate_python(exc.detail)
+    except ValidationError:
+        return str(exc.detail)
+    return str(detail.get("error", detail))
+
+
+def _requested_teams(item: BulkNewUserItem) -> tuple[NewUserRequestTeam, ...]:
+    if item.team_id is not None:
+        return (NewUserRequestTeam(team_id=item.team_id),)
+    teams: Final = item.teams if item.teams is not None else check_if_default_team_set()
+    if teams is None:
+        return ()
+    return tuple(team if isinstance(team, NewUserRequestTeam) else NewUserRequestTeam(team_id=team) for team in teams)
+
+
+def _row_error(item: BulkNewUserItem, user_api_key_dict: UserAPIKeyAuth) -> str | None:
+    if (
+        item.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
+        and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
+    ):
+        return (
+            "Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). "
+            f"Attempted to create user with role: {item.user_role}. Your role: {user_api_key_dict.user_role}"
+        )
+    try:
+        validate_budget_duration(item.budget_duration)
+        _check_permissions_caller_permission(data=item, user_api_key_dict=user_api_key_dict)
+    except Exception as exc:  # noqa: BLE001  # any validation failure is reported on this row only
+        return _error_message(exc)
+    return None
+
+
+def _normalized_email(email: str | None) -> str | None:
+    return email.strip().lower() if email else None
+
+
+def _partition_rows(
+    users: Sequence[BulkNewUserItem], user_api_key_dict: UserAPIKeyAuth
+) -> tuple[tuple[_PendingUser, ...], tuple[_RowFailure, ...]]:
+    """Assign ids, run the per-row checks and fail later rows that repeat an earlier row's id or email."""
+    user_ids: Final = tuple(item.user_id or str(uuid.uuid4()) for item in users)
+    first_index_by_id: Final = MappingProxyType(
+        {user_id: index for index, user_id in reversed(tuple(enumerate(user_ids)))}
+    )
+    first_index_by_email: Final = MappingProxyType(
+        {
+            email: index
+            for index, email in reversed(tuple(enumerate(_normalized_email(item.user_email) for item in users)))
+            if email is not None
+        }
+    )
+
+    def classify(index: int, item: BulkNewUserItem) -> _PendingUser | _RowFailure:
+        user_id: Final = user_ids[index]
+        email: Final = _normalized_email(item.user_email)
+        if first_index_by_id[user_id] != index:
+            return _RowFailure(index, user_id, item.user_email, f"Duplicate user_id in request: {user_id}")
+        if email is not None and first_index_by_email[email] != index:
+            return _RowFailure(index, user_id, item.user_email, f"Duplicate user_email in request: {item.user_email}")
+        error: Final = _row_error(item, user_api_key_dict)
+        if error is not None:
+            return _RowFailure(index, user_id, item.user_email, error)
+        return _PendingUser(index, item, user_id, _requested_teams(item))
+
+    outcomes: Final = tuple(classify(index, item) for index, item in enumerate(users))
+    return (
+        tuple(outcome for outcome in outcomes if isinstance(outcome, _PendingUser)),
+        tuple(outcome for outcome in outcomes if isinstance(outcome, _RowFailure)),
+    )
+
+
+def _user_table(prisma_client: PrismaClient) -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    return UserRepository(prisma_client).table
+
+
+async def _existing_user_conflicts(
+    prisma_client: PrismaClient, pending: Sequence[_PendingUser]
+) -> tuple[frozenset[str], frozenset[str]]:
+    """Return the requested user ids and (lowercased) emails that already exist, using one query each."""
+    user_ids: Final = sorted(user.user_id for user in pending)
+    emails: Final = sorted(frozenset(user.request.user_email for user in pending if user.request.user_email))
+    if not user_ids:
+        return frozenset(), frozenset()
+    table: Final = _user_table(prisma_client)
+    id_filter: Final = {"user_id": {"in": user_ids}}  # mutable-ok: Prisma query filters are dict-shaped
+    email_filter: Final = {"user_email": {"in": emails, "mode": "insensitive"}}  # mutable-ok: Prisma filter
+    id_rows: Final = await table.find_many(where=id_filter)
+    email_rows: Final = await table.find_many(where=email_filter) if emails else ()
+    return (
+        frozenset(row.user_id for row in id_rows),
+        frozenset(lowered for row in email_rows if (lowered := _normalized_email(row.user_email)) is not None),
+    )
+
+
+async def _load_teams(prisma_client: PrismaClient, team_ids: frozenset[str]) -> Mapping[str, LiteLLM_TeamTable]:
+    if not team_ids:
+        return MappingProxyType({})
+    rows: Final = await TeamRepository(prisma_client).table.find_many(
+        where={"team_id": {"in": sorted(team_ids)}}  # mutable-ok: Prisma query filters are dict-shaped
+    )
+    return MappingProxyType({row.team_id: LiteLLM_TeamTable.model_validate(row.model_dump()) for row in rows})
+
+
+async def _team_permission_error(team: LiteLLM_TeamTable, user_api_key_dict: UserAPIKeyAuth) -> str | None:
+    if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value:
+        return None
+    if _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team):
+        return None
+    if await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team):
+        return None
+    return f"Call not allowed. User not proxy admin OR team admin. team_id={team.team_id}"
+
+
+async def _unusable_teams(
+    prisma_client: PrismaClient,
+    pending: Sequence[_PendingUser],
+    user_api_key_dict: UserAPIKeyAuth,
+) -> tuple[Mapping[str, LiteLLM_TeamTable], Mapping[str, str]]:
+    """Load every referenced team once and explain, per team id, why rows naming it cannot proceed."""
+    team_ids: Final = frozenset(team.team_id for user in pending for team in user.teams)
+    teams: Final = await _load_teams(prisma_client, team_ids)
+    permission_errors: Final = await asyncio.gather(
+        *(_team_permission_error(team, user_api_key_dict) for team in teams.values())
+    )
+    missing: Final = tuple(
+        (team_id, f"Team id={team_id} does not exist") for team_id in team_ids if team_id not in teams
+    )
+    denied: Final = tuple(
+        (team.team_id, error)
+        for team, error in zip(teams.values(), permission_errors, strict=True)
+        if error is not None
+    )
+    return teams, MappingProxyType({team_id: error for team_id, error in (*missing, *denied)})
+
+
+def _db_failure(
+    user: _PendingUser,
+    existing_ids: frozenset[str],
+    existing_emails: frozenset[str],
+    team_errors: Mapping[str, str],
+) -> _RowFailure | None:
+    email: Final = _normalized_email(user.request.user_email)
+    if user.user_id in existing_ids:
+        return _RowFailure(user.index, user.user_id, user.request.user_email, f"User id={user.user_id} already exists")
+    if email is not None and email in existing_emails:
+        return _RowFailure(
+            user.index, user.user_id, user.request.user_email, f"User email={user.request.user_email} already exists"
+        )
+    errors: Final = tuple(team_errors[team.team_id] for team in user.teams if team.team_id in team_errors)
+    if errors:
+        return _RowFailure(user.index, user.user_id, user.request.user_email, "; ".join(errors))
+    return None
+
+
+async def _prepare_user(user: _PendingUser, prisma_client: PrismaClient) -> _PreparedUser | _RowFailure:
+    try:
+        dumped: Final = user.request.model_dump(exclude={"user_id"})  # mutable-ok: pydantic IncEx takes a set
+        data: Final = {**dumped, "user_id": user.user_id}  # mutable-ok: /user/new defaults helper mutates in place
+        data_json: Final = _JSON_OBJECT.validate_python(_update_internal_new_user_params(data, user.request))
+        with_permission: Final = _JSON_OBJECT.validate_python(
+            await _set_object_permission(data_json=data_json, prisma_client=prisma_client)  # pyright: ignore[reportUnknownArgumentType]  # validated by the adapter
+        )
+        return _PreparedUser(user, _USER_ROW.validate_python(with_permission))
+    except Exception as exc:  # noqa: BLE001  # any preparation failure is reported on this row only
+        verbose_proxy_logger.warning("/user/bulk_new: could not prepare row %d - %s", user.index, type(exc).__name__)
+        return _RowFailure(user.index, user.user_id, user.request.user_email, _error_message(exc))
+
+
+class _UserCreateData(TypedDict):
+    """One `LiteLLM_UserTable` row as `create_many` takes it; JSON columns are pre-serialized."""
+
+    user_id: ReadOnly[str]
+    user_email: ReadOnly[str | None]
+    user_alias: ReadOnly[str | None]
+    user_role: ReadOnly[str | None]
+    team_id: ReadOnly[str | None]
+    max_budget: ReadOnly[float | None]
+    spend: ReadOnly[float]
+    models: ReadOnly[tuple[str, ...]]
+    metadata: ReadOnly[str]
+    max_parallel_requests: ReadOnly[int | None]
+    tpm_limit: ReadOnly[int | None]
+    rpm_limit: ReadOnly[int | None]
+    budget_duration: ReadOnly[str | None]
+    budget_reset_at: ReadOnly[datetime | None]
+    allowed_cache_controls: ReadOnly[tuple[str, ...]]
+    sso_user_id: ReadOnly[str | None]
+    object_permission_id: ReadOnly[str | None]
+    teams: ReadOnly[tuple[str, ...]]
+    model_max_budget: ReadOnly[str]
+
+
+def _user_create_payload(prepared: _PreparedUser) -> _UserCreateData:
+    row: Final = prepared.row
+    metadata_json: Final = metadata_json_with_limits(
+        row.metadata,
+        model_rpm_limit=row.model_rpm_limit,
+        model_tpm_limit=row.model_tpm_limit,
+        mcp_rpm_limit=row.mcp_rpm_limit,
+        tag_rpm_limit=row.tag_rpm_limit,
+        guardrails=row.guardrails,
+        policies=row.policies,
+        prompts=row.prompts,
+    )
+    payload: Final[_UserCreateData] = {
+        "user_id": row.user_id,
+        "user_email": row.user_email,
+        "user_alias": row.user_alias,
+        "user_role": row.user_role,
+        "team_id": row.team_id,
+        "max_budget": row.max_budget,
+        "spend": row.spend or 0.0,
+        "models": row.models or (),
+        "metadata": metadata_json,
+        "max_parallel_requests": row.max_parallel_requests,
+        "tpm_limit": row.tpm_limit,
+        "rpm_limit": row.rpm_limit,
+        "budget_duration": row.budget_duration,
+        "budget_reset_at": get_budget_reset_time(row.budget_duration) if row.budget_duration else None,
+        "allowed_cache_controls": row.allowed_cache_controls or (),
+        "sso_user_id": row.sso_user_id,
+        "object_permission_id": row.object_permission_id,
+        "teams": tuple(team.team_id for team in prepared.pending.teams),
+        "model_max_budget": json.dumps(row.model_max_budget) if row.model_max_budget else "{}",
+    }
+    return payload
+
+
+async def _bounded(limit: int, awaitables: Sequence[Awaitable[_T]]) -> tuple[_T | BaseException, ...]:
+    semaphore: Final = asyncio.Semaphore(limit)
+
+    async def run(awaitable: Awaitable[_T]) -> _T:
+        async with semaphore:
+            return await awaitable
+
+    return tuple(await asyncio.gather(*(run(awaitable) for awaitable in awaitables), return_exceptions=True))
+
+
+async def _insert_users(
+    prisma_client: PrismaClient, prepared: Sequence[_PreparedUser]
+) -> tuple[tuple[_PreparedUser, ...], tuple[_RowFailure, ...]]:
+    """Insert every row in one statement. If that fails, retry rows one at a time so the error lands on its row."""
+    if not prepared:
+        return (), ()
+    table: Final = _user_table(prisma_client)
+    payloads: Final = tuple(_user_create_payload(user) for user in prepared)
+    try:
+        await table.create_many(data=payloads)
+        return tuple(prepared), ()
+    except Exception as exc:  # noqa: BLE001  # fall back to per-row inserts so the failing row can be identified
+        verbose_proxy_logger.warning("/user/bulk_new: create_many failed, retrying rows individually", exc_info=True)
+        outcome_unknown: Final = PrismaDBExceptionHandler.is_database_infrastructure_error(exc)
+    requested: Final = frozenset(payload["user_id"] for payload in payloads)
+    landed_rows: Final = await table.find_many(where={"user_id": {"in": list(requested)}})  # mutable-ok: Prisma filter
+    landed: Final = frozenset(row.user_id for row in landed_rows)
+    # create_many is one INSERT: after a lost response the full set is ours, any partial set belongs to another request
+    if outcome_unknown and landed == requested:
+        return tuple(prepared), ()
+    taken: Final = tuple(user for user in prepared if user.row.user_id in landed)
+    retried: Final = tuple(user for user in prepared if user.row.user_id not in landed)
+    outcomes: Final = await _bounded(
+        BULK_NEW_USER_CONCURRENCY, tuple(table.create(data=_user_create_payload(user)) for user in retried)
+    )
+    failed: Final = MappingProxyType(
+        {
+            **{
+                user.row.user_id: _RowFailure(
+                    user.pending.index,
+                    user.pending.user_id,
+                    user.row.user_email,
+                    f"User id={user.row.user_id} already exists",
+                )
+                for user in taken
+            },
+            **{
+                user.row.user_id: _RowFailure(
+                    user.pending.index, user.pending.user_id, user.row.user_email, _error_message(outcome)
+                )
+                for user, outcome in zip(retried, outcomes, strict=True)
+                if isinstance(outcome, BaseException)
+            },
+        }
+    )
+    return (
+        tuple(user for user in prepared if user.row.user_id not in failed),
+        tuple(failed.values()),
+    )
+
+
+def _assignments_by_team(created: Sequence[_PreparedUser]) -> Mapping[str, tuple[_TeamAssignment, ...]]:
+    team_ids: Final = tuple(dict.fromkeys(team.team_id for user in created for team in user.pending.teams))
+    return MappingProxyType(
+        {
+            team_id: tuple(
+                _TeamAssignment(user.pending.user_id, user.row.user_email, team.user_role, team.max_budget_in_team)
+                for user in created
+                for team in user.pending.teams
+                if team.team_id == team_id
+            )
+            for team_id in team_ids
+        }
+    )
+
+
+class _MembershipData(TypedDict):
+    team_id: ReadOnly[str]
+    user_id: ReadOnly[str]
+    budget_id: ReadOnly[str | None]
+
+
+class _RosterData(TypedDict):
+    members_with_roles: ReadOnly[str]
+
+
+class _TeamsData(TypedDict):
+    teams: ReadOnly[tuple[str, ...]]
+
+
+def _default_member_budget_id(team: LiteLLM_TeamTable) -> str | None:
+    metadata: Final = (
+        _JSON_OBJECT.validate_python(
+            team.metadata  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # LiteLLM_TeamTable.metadata is a bare dict; validated by the adapter
+        )
+        if team.metadata  # pyright: ignore[reportUnknownMemberType]  # same bare dict
+        else None
+    )
+    budget_id: Final = metadata.get("team_member_budget_id") if metadata is not None else None
+    return budget_id if isinstance(budget_id, str) else None
+
+
+def _team_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    return tx.litellm_teamtable  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+def _membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamMembership]":
+    return tx.litellm_teammembership  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+async def _write_team_roster(
+    prisma_client: PrismaClient,
+    team: LiteLLM_TeamTable,
+    members: Sequence[_TeamAssignment],
+    user_api_key_dict: UserAPIKeyAuth,
+    litellm_proxy_admin_name: str,
+) -> _TeamWrite:
+    """Add every new member to one team under its advisory lock: one roster rewrite and one membership insert."""
+    try:
+        async with prisma_client.tx() as tx:
+            await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team.team_id)
+            roster: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, team.team_id)
+            if roster is None:
+                raise ValueError(f"Team id={team.team_id} does not exist")
+            already_present: Final = frozenset(member.user_id for member in roster if member.user_id)
+            new_members: Final = tuple(member for member in members if member.user_id not in already_present)
+            budget_ids: Final = tuple(
+                [  # mutable-ok: budgets are created one at a time on the transaction's single connection
+                    await _resolve_member_budget_id(
+                        prisma_client=prisma_client,
+                        user_api_key_dict=user_api_key_dict,
+                        litellm_proxy_admin_name=litellm_proxy_admin_name,
+                        max_budget_in_team=member.max_budget_in_team,
+                        allowed_models=team.default_team_member_models or None,
+                        budget_duration=None,
+                        default_team_budget_id=_default_member_budget_id(team),
+                        tx=tx,  # pyright: ignore[reportArgumentType]  # MemberWriteTx lags the generated Prisma signatures, same as /team/member_add
+                    )
+                    for member in new_members
+                ]
+            )
+            await _membership_tx_db(tx).create_many(
+                data=tuple(
+                    _MembershipData(team_id=team.team_id, user_id=member.user_id, budget_id=budget_id)
+                    for member, budget_id in zip(new_members, budget_ids, strict=True)
+                ),
+                skip_duplicates=True,
+            )
+            after: Final = (
+                *roster,
+                *(Member(user_id=m.user_id, user_email=m.user_email, role=m.role) for m in new_members),
+            )
+            await _team_tx_db(tx).update(
+                where={"team_id": team.team_id},  # mutable-ok: Prisma query filters are dict-shaped
+                data=_RosterData(members_with_roles=json.dumps(tuple(member.model_dump() for member in after))),
+            )
+        return _TeamWrite(
+            team_id=team.team_id,
+            after=after,
+            added=frozenset(member.user_id for member in members),
+            failed=MappingProxyType({}),
+        )
+    except Exception as exc:  # noqa: BLE001  # the team write failure is reported on each affected row
+        verbose_proxy_logger.exception("/user/bulk_new: failed to add %d members to a team", len(members))
+        message: Final = f"Failed to add user to team {team.team_id}: {_error_message(exc)}"
+        return _TeamWrite(
+            team_id=team.team_id,
+            after=(),
+            added=frozenset(),
+            failed=MappingProxyType({member.user_id: message for member in members}),
+        )
+
+
+async def _detach_failed_teams(
+    prisma_client: PrismaClient, created: Sequence[_PreparedUser], writes: Mapping[str, _TeamWrite]
+) -> None:
+    """Users are inserted with `teams` already set; drop the teams whose roster write did not take them."""
+    table: Final = _user_table(prisma_client)
+    updates: Final = tuple(
+        table.update(
+            where={"user_id": user.row.user_id},  # mutable-ok: Prisma query filters are dict-shaped
+            data=_TeamsData(teams=landed),
+        )
+        for user in created
+        if (landed := _row_teams(user, writes)[0]) != tuple(team.team_id for team in user.pending.teams)
+    )
+    for outcome in await _bounded(BULK_NEW_USER_CONCURRENCY, updates):
+        if isinstance(outcome, BaseException):
+            verbose_proxy_logger.warning(
+                "/user/bulk_new: could not detach failed teams from user - %s", type(outcome).__name__
+            )
+
+
+async def _publish_team_writes(writes: Sequence[_TeamWrite], user_api_key_cache: "UserApiKeyCache") -> None:
+    prometheus_logger: Final = PrometheusLogger.get_instance()
+    for write in writes:
+        if prometheus_logger is None or not write.added:
+            continue
+        try:
+            prometheus_logger.set_team_members_metric(
+                LiteLLM_TeamTable(
+                    team_id=write.team_id,
+                    members_with_roles=write.after,  # pyright: ignore[reportArgumentType]  # pydantic coerces the tuple into the declared list
+                )
+            )
+        except Exception:  # noqa: BLE001  # metrics are best-effort and must not fail the request
+            verbose_proxy_logger.debug("Prometheus: failed to emit team members metric", exc_info=True)
+    evictions: Final = await _bounded(
+        BULK_NEW_USER_CONCURRENCY,
+        tuple(
+            invalidate_team_member_spend_state(
+                user_id=user_id, team_id=write.team_id, user_api_key_cache=user_api_key_cache
+            )
+            for write in writes
+            for user_id in write.added
+        ),
+    )
+    for eviction in evictions:
+        if isinstance(eviction, BaseException):
+            verbose_proxy_logger.warning("/user/bulk_new: cache eviction failed - %s", type(eviction).__name__)
+
+
+_KEY_FIELDS: Final = MappingProxyType(
+    {
+        name: True
+        for name in (
+            "user_id",
+            "team_id",
+            "agent_id",
+            "duration",
+            "key_alias",
+            "models",
+            "aliases",
+            "config",
+            "permissions",
+            "blocked",
+            "spend",
+            "budget_fallbacks",
+            "budget_limits",
+            "metadata",
+            "max_parallel_requests",
+            "tpm_limit",
+            "rpm_limit",
+            "allowed_cache_controls",
+            "model_max_budget",
+            "model_rpm_limit",
+            "model_tpm_limit",
+            "mcp_rpm_limit",
+            "tag_rpm_limit",
+            "guardrails",
+            "policies",
+            "prompts",
+            "object_permission_id",
+        )
+    }
+)
+
+
+async def _generate_key(prepared: _PreparedUser, generate_key: KeyGenerator) -> str:
+    response: Final = _KEY_RESPONSE.validate_python(
+        await generate_key(
+            request_type="key", table_name="key", **prepared.row.model_dump(include=_KEY_FIELDS, exclude_none=True)
+        )
+    )
+    return response.token
+
+
+async def _add_to_organizations(
+    prepared: _PreparedUser, organizations: Sequence[str], user_api_key_dict: UserAPIKeyAuth
+) -> None:
+    for organization_id in organizations:
+        await organization_member_add(
+            data=OrganizationMemberAddRequest(
+                organization_id=organization_id,
+                member=OrgMember(user_id=prepared.row.user_id, role=LitellmUserRoles.INTERNAL_USER),
+            ),
+            http_request=Request(scope={"type": "http", "path": "/user/bulk_new"}),  # mutable-ok: ASGI scopes are dicts
+            user_api_key_dict=user_api_key_dict,
+        )
+
+
+async def _run_per_user(
+    created: Sequence[_PreparedUser],
+    select: Callable[[_PreparedUser], bool],
+    action: Callable[[_PreparedUser], Awaitable[_T]],
+) -> Mapping[str, _T | BaseException]:
+    chosen: Final = tuple(user for user in created if select(user))
+    outcomes: Final = await _bounded(BULK_NEW_USER_CONCURRENCY, tuple(action(user) for user in chosen))
+    return MappingProxyType({user.row.user_id: outcome for user, outcome in zip(chosen, outcomes, strict=True)})
+
+
+async def _write_audit_logs(
+    prisma_client: PrismaClient,
+    created: Sequence[_PreparedUser],
+    user_api_key_dict: UserAPIKeyAuth,
+    litellm_proxy_admin_name: str,
+) -> None:
+    if not created:
+        return
+    created_ids: Final = sorted(user.row.user_id for user in created)
+    created_filter: Final = {"user_id": {"in": created_ids}}  # mutable-ok: Prisma query filters are dict-shaped
+    rows: Final = await _user_table(prisma_client).find_many(where=created_filter)
+    outcomes: Final = await _bounded(
+        BULK_NEW_USER_CONCURRENCY,
+        tuple(
+            UserManagementEventHooks.create_internal_user_audit_log(
+                user_id=row.user_id,
+                action="created",
+                litellm_changed_by=user_api_key_dict.user_id,
+                user_api_key_dict=user_api_key_dict,
+                litellm_proxy_admin_name=litellm_proxy_admin_name,
+                before_value=None,
+                after_value=row.model_dump_json(exclude_none=True),
+            )
+            for row in rows
+        ),
+    )
+    for outcome in outcomes:
+        if isinstance(outcome, BaseException):
+            verbose_proxy_logger.warning(
+                "Unable to create audit log for user on `/user/bulk_new` - %s", type(outcome).__name__
+            )
+
+
+def _row_teams(prepared: _PreparedUser, writes: Mapping[str, _TeamWrite]) -> tuple[tuple[str, ...], tuple[str, ...]]:
+    """Split a user's requested teams into the ones they landed in and the errors for the ones they did not."""
+    requested: Final = tuple(team.team_id for team in prepared.pending.teams)
+    return (
+        tuple(team_id for team_id in requested if prepared.row.user_id in writes[team_id].added),
+        tuple(
+            writes[team_id].failed[prepared.row.user_id]
+            for team_id in requested
+            if prepared.row.user_id in writes[team_id].failed
+        ),
+    )
+
+
+def _to_result(created: _CreatedUser) -> UserCreateResult:
+    return UserCreateResult(
+        user_id=created.prepared.row.user_id,
+        user_email=created.prepared.row.user_email,
+        success=True,
+        teams=created.teams,
+        key=created.key,
+        error="; ".join(created.errors) if created.errors else None,
+    )
+
+
+def _failure_result(failure: _RowFailure) -> UserCreateResult:
+    return UserCreateResult(user_id=failure.user_id, user_email=failure.user_email, success=False, error=failure.error)
+
+
+async def bulk_create_users(
+    users: Sequence[BulkNewUserItem],
+    user_api_key_dict: UserAPIKeyAuth,
+    prisma_client: PrismaClient,
+    license_check: LicenseCheck,
+    litellm_proxy_admin_name: str,
+    user_api_key_cache: "UserApiKeyCache",
+    generate_key: KeyGenerator = generate_key_helper_fn,
+) -> BulkNewUserResponse:
+    """Create every valid row in `users`; rows that fail validation or a write are reported, not raised.
+
+    Raises a 403 `ManagementProblem` only when the whole batch would push the deployment over its license seat
+    limit.
+    """
+    pending, request_failures = _partition_rows(users, user_api_key_dict)
+    existing_ids, existing_emails = await _existing_user_conflicts(prisma_client, pending)
+    teams, team_errors = await _unusable_teams(prisma_client, pending, user_api_key_dict)
+    db_failures: Final = tuple(
+        failure
+        for user in pending
+        if (failure := _db_failure(user, existing_ids, existing_emails, team_errors)) is not None
+    )
+    failed_indexes: Final = frozenset(failure.index for failure in db_failures)
+    creatable: Final = tuple(user for user in pending if user.index not in failed_indexes)
+
+    billable_users: Final = await UserRepository(prisma_client).count_billable_users()
+    if creatable and license_check.is_over_limit(total_users=billable_users + len(creatable)):
+        raise ManagementProblem(
+            ProblemDetail(
+                type=f"{PROBLEM_TYPE_BASE}license-limit-exceeded",
+                title="License limit exceeded",
+                status=403,
+                detail="License is over limit. Please contact support@berri.ai to upgrade your license.",
+            )
+        )
+
+    prepared_outcomes: Final = tuple([await _prepare_user(user, prisma_client) for user in creatable])
+    prepare_failures: Final = tuple(o for o in prepared_outcomes if isinstance(o, _RowFailure))
+    created, insert_failures = await _insert_users(
+        prisma_client, tuple(o for o in prepared_outcomes if isinstance(o, _PreparedUser))
+    )
+
+    team_writes: Final = MappingProxyType(
+        {
+            team_id: await _write_team_roster(
+                prisma_client, teams[team_id], members, user_api_key_dict, litellm_proxy_admin_name
+            )
+            for team_id, members in _assignments_by_team(created).items()
+        }
+    )
+    await _detach_failed_teams(prisma_client, created, team_writes)
+    await _publish_team_writes(tuple(team_writes.values()), user_api_key_cache)
+
+    keys: Final = await _run_per_user(
+        created, lambda user: user.pending.request.auto_create_key, lambda user: _generate_key(user, generate_key)
+    )
+    org_outcomes: Final = await _run_per_user(
+        created,
+        lambda user: bool(user.row.organizations),
+        lambda user: _add_to_organizations(user, user.row.organizations or (), user_api_key_dict),
+    )
+    await _write_audit_logs(prisma_client, created, user_api_key_dict, litellm_proxy_admin_name)
+
+    def finish(prepared: _PreparedUser) -> _CreatedUser:
+        landed, team_failures = _row_teams(prepared, team_writes)
+        key_outcome: Final = keys.get(prepared.row.user_id)
+        org_outcome: Final = org_outcomes.get(prepared.row.user_id)
+        return _CreatedUser(
+            prepared=prepared,
+            teams=landed,
+            key=key_outcome if isinstance(key_outcome, str) else None,
+            errors=(
+                *team_failures,
+                *(
+                    (f"Failed to create key: {_error_message(key_outcome)}",)
+                    if isinstance(key_outcome, BaseException)
+                    else ()
+                ),
+                *(
+                    (f"Failed to add user to organizations: {_error_message(org_outcome)}",)
+                    if isinstance(org_outcome, BaseException)
+                    else ()
+                ),
+            ),
+        )
+
+    failures: Final = MappingProxyType(
+        {
+            failure.index: _failure_result(failure)
+            for failure in (*request_failures, *db_failures, *prepare_failures, *insert_failures)
+        }
+    )
+    successes_by_index: Final = MappingProxyType({user.pending.index: _to_result(finish(user)) for user in created})
+    results: Final = tuple(
+        failures[index] if index in failures else successes_by_index[index] for index in range(len(users))
+    )
+    successes: Final = sum(1 for result in results if result.success)
+    return BulkNewUserResponse(
+        data=results,
+        meta=BulkNewUserMeta(total_requested=len(users), created=successes, failed=len(users) - successes),
+    )

litellm/proxy/management_helpers/bulk_user_deletion.py

@@ -0,0 +1,560 @@
+"""Batched deletes behind `POST /management/v1/users/bulk_delete` and
+`POST /management/v1/teams/{team_id}/members/bulk_delete`.
+
+Each team a batch touches is rewritten exactly once, under the same advisory lock
+`/team/member_delete` takes and from a roster re-read under that lock, so a concurrent
+member_add on the team is never overwritten from a stale read. A user batch runs in one
+transaction, taking its team locks in sorted order, so either every team rewrite and every
+user row delete lands or none of them does.
+"""
+
+import asyncio
+import json
+from collections.abc import Awaitable, Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import timedelta
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final
+
+from fastapi import HTTPException
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm._logging import verbose_proxy_logger
+from litellm.integrations.prometheus import PrometheusLogger
+from litellm.proxy._types import (
+    LiteLLM_TeamTable,
+    LitellmUserRoles,
+    Member,
+    MemberDeleteRequest,
+    UserAPIKeyAuth,
+)
+from litellm.proxy.auth.auth_checks import delete_cache_key_objects
+from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks
+from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem
+from litellm.proxy.management_endpoints.common_utils import (
+    _is_user_org_admin_for_team,  # pyright: ignore[reportPrivateUsage]  # same check /team/member_delete uses
+    _is_user_team_admin,  # pyright: ignore[reportPrivateUsage]  # same check /team/member_delete uses
+)
+from litellm.proxy.management_endpoints.key_management_endpoints import (
+    _persist_deleted_verification_tokens,  # pyright: ignore[reportPrivateUsage]  # same audit path /key/delete uses
+)
+from litellm.proxy.management_helpers.access_group_team_sync import TEAM_ADVISORY_LOCK_SQL
+from litellm.proxy.utils import PrismaClient, ProxyLogging
+from litellm.repositories.table_repositories import (
+    OrganizationMembershipRepository,
+    TeamMembershipRepository,
+)
+from litellm.repositories.team_repository import TeamRepository
+from litellm.repositories.user_repository import UserRepository
+from litellm.types.proxy.management_endpoints.internal_user_endpoints import (
+    BulkDeleteUserRequest,
+    UserDeleteResult,
+)
+from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail
+from litellm.types.proxy.management_endpoints.team_endpoints import (
+    BulkTeamMemberDeleteRequest,
+    TeamMemberDeleteResult,
+)
+
+if TYPE_CHECKING:
+    from prisma import Prisma
+    from prisma import models as prisma_models
+
+    from litellm.repositories.prisma_protocols import TableActions
+
+_AUDIT_LOG_CONCURRENCY: Final = 10
+_BATCH_TX_TIMEOUT: Final = timedelta(seconds=60)
+
+
+class _OrgAdminFilter(TypedDict):
+    user_id: ReadOnly[str]
+    user_role: ReadOnly[str]
+
+
+class _RosterData(TypedDict):
+    members_with_roles: ReadOnly[str]
+
+
+class _TeamsSet(TypedDict):
+    set: ReadOnly[tuple[str, ...]]
+
+
+class _TeamsData(TypedDict):
+    teams: ReadOnly[_TeamsSet]
+
+
+@dataclass(frozen=True, slots=True)
+class _TeamRemoval:
+    """One team's rewrite. `removed` holds the user ids taken off the team (roster, `teams` array, or both);
+    `matched` holds the indexes into the requested members that named at least one of them."""
+
+    team: LiteLLM_TeamTable
+    removed: frozenset[str]
+    matched: frozenset[int]
+    deleted_key_tokens: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class _UserBatchDeletion:
+    removals: Mapping[str, _TeamRemoval]
+    deleted_key_tokens: tuple[str, ...]
+
+
+def _team_not_found(team_id: str) -> ManagementProblem:
+    return ManagementProblem(
+        ProblemDetail(
+            type=f"{PROBLEM_TYPE_BASE}team-not-found",
+            title="Team not found",
+            status=404,
+            detail=f"Team id={team_id} does not exist in db",
+        )
+    )
+
+
+def _forbidden(detail: str) -> ManagementProblem:
+    return ManagementProblem(
+        ProblemDetail(type=f"{PROBLEM_TYPE_BASE}forbidden", title="Forbidden", status=403, detail=detail)
+    )
+
+
+def _in_filter(field: str, values: Iterable[str]) -> Mapping[str, object]:
+    return {field: {"in": sorted(values)}}  # mutable-ok: Prisma query filters are dict-shaped
+
+
+def _eq_filter(field: str, value: str) -> Mapping[str, object]:
+    return {field: value}  # mutable-ok: Prisma query filters are dict-shaped
+
+
+def _team_users_filter(team_id: str, user_ids: Iterable[str]) -> Mapping[str, object]:
+    return {"team_id": team_id, **_in_filter("user_id", user_ids)}  # mutable-ok: Prisma query filters are dict-shaped
+
+
+def _any_filter(*clauses: Mapping[str, object]) -> Mapping[str, object]:
+    return {"OR": clauses}  # mutable-ok: Prisma query filters are dict-shaped
+
+
+def _team_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    return tx.litellm_teamtable  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+def _user_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    return tx.litellm_usertable  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+def _membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamMembership]":
+    return tx.litellm_teammembership  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+def _token_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
+    return tx.litellm_verificationtoken  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+def _invitation_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_InvitationLink]":
+    return tx.litellm_invitationlink  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+def _org_membership_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_OrganizationMembership]":
+    return tx.litellm_organizationmembership  # pyright: ignore[reportReturnType]  # TableActions widens the generated inputs to Mapping, as the repositories do
+
+
+def _same_email(email: str | None, request: MemberDeleteRequest) -> bool:
+    return request.user_email is not None and request.user_email == email
+
+
+def _addresses_member(member: Member, request: MemberDeleteRequest) -> bool:
+    if request.user_id is None:
+        return _same_email(member.user_email, request)
+    return request.user_id == member.user_id or (member.user_id is None and _same_email(member.user_email, request))
+
+
+def _with_row_email(request: MemberDeleteRequest, email_of: Mapping[str, str]) -> MemberDeleteRequest:
+    if request.user_id is None or request.user_email is not None:
+        return request
+    return MemberDeleteRequest(user_id=request.user_id, user_email=email_of.get(request.user_id))
+
+
+def _addresses_user(user: "prisma_models.LiteLLM_UserTable", request: MemberDeleteRequest) -> bool:
+    if request.user_id is None:
+        return _same_email(user.user_email, request)
+    return request.user_id == user.user_id
+
+
+def _error_message(exc: BaseException) -> str:
+    if isinstance(exc, ManagementProblem):
+        return exc.problem.detail
+    if isinstance(exc, HTTPException) and isinstance(exc.detail, dict):
+        return str(exc.detail.get("error", exc.detail))  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # HTTPException.detail is untyped
+    if isinstance(exc, HTTPException):
+        return str(exc.detail)  # pyright: ignore[reportUnknownArgumentType]  # HTTPException.detail is untyped
+    return str(exc) or type(exc).__name__
+
+
+async def _bounded(awaitables: Iterable[Awaitable[object]]) -> tuple[object | BaseException, ...]:
+    semaphore: Final = asyncio.Semaphore(_AUDIT_LOG_CONCURRENCY)
+
+    async def run(awaitable: Awaitable[object]) -> object:
+        async with semaphore:
+            return await awaitable
+
+    return tuple(await asyncio.gather(*(run(a) for a in awaitables), return_exceptions=True))
+
+
+async def _remove_members_from_team(
+    prisma_client: PrismaClient,
+    tx: "Prisma",
+    team_id: str,
+    members: Sequence[MemberDeleteRequest],
+    user_api_key_dict: UserAPIKeyAuth,
+) -> _TeamRemoval:
+    await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id)
+    roster: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, team_id)
+    if roster is None:
+        raise _team_not_found(team_id)
+
+    requested_ids: Final = frozenset(r.user_id for r in members if r.user_id is not None)
+    requested_emails: Final = frozenset(r.user_email for r in members if r.user_id is None and r.user_email)
+    requested_rows: Final = await _user_tx_db(tx).find_many(
+        where=_any_filter(_in_filter("user_id", requested_ids), _in_filter("user_email", requested_emails))
+    )
+    email_of: Final = MappingProxyType(
+        {u.user_id: u.user_email for u in requested_rows if u.user_email is not None and team_id in u.teams}
+    )
+    requests: Final = tuple(_with_row_email(r, email_of) for r in members)
+    removed_members: Final = tuple(m for m in roster if any(_addresses_member(m, r) for r in requests))
+    kept_members: Final = tuple(m for m in roster if not any(_addresses_member(m, r) for r in requests))
+    removed_ids: Final = frozenset(m.user_id for m in removed_members if m.user_id is not None)
+    unfetched_ids: Final = removed_ids - frozenset(u.user_id for u in requested_rows)
+    removed_rows: Final = (
+        await _user_tx_db(tx).find_many(where=_in_filter("user_id", unfetched_ids)) if unfetched_ids else ()
+    )
+    stale_rows: Final = tuple(u for u in (*requested_rows, *removed_rows) if team_id in u.teams)
+    cleanup_ids: Final = removed_ids | frozenset(u.user_id for u in stale_rows)
+    matched: Final = frozenset(
+        i
+        for i, r in enumerate(requests)
+        if any(_addresses_member(m, r) for m in removed_members) or any(_addresses_user(u, r) for u in stale_rows)
+    )
+    keys: Final = await _token_tx_db(tx).find_many(where=_team_users_filter(team_id, cleanup_ids))
+
+    if removed_members:
+        roster_data: Final[_RosterData] = {
+            "members_with_roles": json.dumps(tuple(m.model_dump() for m in kept_members))
+        }
+        await _team_tx_db(tx).update(where=_eq_filter("team_id", team_id), data=roster_data)
+    for row in stale_rows:
+        teams_data: _TeamsData = {"teams": {"set": tuple(t for t in row.teams if t != team_id)}}
+        await _user_tx_db(tx).update(where=_eq_filter("user_id", row.user_id), data=teams_data)
+    await _membership_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids))
+    if keys:
+        await _persist_deleted_verification_tokens(
+            keys=keys,  # pyright: ignore[reportArgumentType]  # generated row model carries the same columns as LiteLLM_VerificationToken
+            prisma_client=prisma_client,
+            user_api_key_dict=user_api_key_dict,
+            litellm_changed_by=None,
+            tx=tx,
+        )
+        await _token_tx_db(tx).delete_many(where=_team_users_filter(team_id, cleanup_ids))
+
+    return _TeamRemoval(
+        team=LiteLLM_TeamTable(
+            team_id=team_id,
+            members_with_roles=kept_members,  # pyright: ignore[reportArgumentType]  # pydantic coerces the tuple into the list field
+        ),
+        removed=cleanup_ids,
+        matched=matched,
+        deleted_key_tokens=tuple(k.token for k in keys),
+    )
+
+
+def _emit_team_members_metric(team: LiteLLM_TeamTable) -> None:
+    prometheus_logger: Final = PrometheusLogger.get_instance()
+    if prometheus_logger is None:
+        return
+    try:
+        prometheus_logger.set_team_members_metric(team)
+    except Exception as e:
+        verbose_proxy_logger.debug("Prometheus: failed to emit team members metric: %s", str(e))
+
+
+def _duplicate_member_indexes(members: Sequence[MemberDeleteRequest]) -> frozenset[int]:
+    return frozenset(
+        i
+        for i, m in enumerate(members)
+        if any(
+            (m.user_id is not None and m.user_id == earlier.user_id)
+            or (m.user_email is not None and m.user_email == earlier.user_email)
+            for earlier in members[:i]
+        )
+    )
+
+
+async def bulk_remove_team_members(
+    team_id: str,
+    data: BulkTeamMemberDeleteRequest,
+    user_api_key_dict: UserAPIKeyAuth,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging | None,
+) -> tuple[TeamMemberDeleteResult, ...]:
+    team: Final = await TeamRepository(prisma_client).find_by_id(team_id)
+    if team is None:
+        raise _team_not_found(team_id)
+
+    if (
+        user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
+        and not _is_user_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team)
+        and not await _is_user_org_admin_for_team(user_api_key_dict=user_api_key_dict, team_obj=team)
+    ):
+        raise _forbidden(
+            "Call not allowed. User not proxy admin OR team admin OR org admin for this team. "
+            f"route='/management/v1/teams/{team_id}/members/bulk_delete'"
+        )
+
+    duplicates: Final = _duplicate_member_indexes(data.members)
+    kept_indexes: Final = tuple(i for i in range(len(data.members)) if i not in duplicates)
+    members: Final = tuple(data.members[i] for i in kept_indexes)
+    async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx:
+        removal: Final = await _remove_members_from_team(prisma_client, tx, team_id, members, user_api_key_dict)
+    await delete_cache_key_objects(
+        hashed_tokens=removal.deleted_key_tokens,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    _emit_team_members_metric(removal.team)
+
+    matched: Final = frozenset(kept_indexes[j] for j in removal.matched)
+
+    def error(index: int) -> str | None:
+        if index in duplicates:
+            return "Duplicate member in request"
+        return None if index in matched else "User not found in team"
+
+    return tuple(
+        TeamMemberDeleteResult(
+            user_id=member.user_id,
+            user_email=member.user_email,
+            success=i in matched,
+            error=error(i),
+        )
+        for i, member in enumerate(data.members)
+    )
+
+
+async def _caller_admin_org_ids(prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]:
+    if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value or not user_api_key_dict.user_id:
+        return frozenset()
+    where: Final[_OrgAdminFilter] = {
+        "user_id": user_api_key_dict.user_id,
+        "user_role": LitellmUserRoles.ORG_ADMIN.value,
+    }
+    memberships: Final = await OrganizationMembershipRepository(prisma_client).table.find_many(where=where)
+    return frozenset(m.organization_id for m in memberships if m.organization_id)
+
+
+def _scope_error(user_id: str, target_org_ids: frozenset[str], caller_admin_org_ids: frozenset[str]) -> str | None:
+    if target_org_ids and target_org_ids <= caller_admin_org_ids:
+        return None
+    return (
+        f"User {user_id} is not within your admin scope. "
+        "Only PROXY_ADMIN may delete users outside your administered organizations."
+    )
+
+
+async def _delete_user_rows(
+    prisma_client: PrismaClient,
+    tx: "Prisma",
+    user_ids: frozenset[str],
+    user_api_key_dict: UserAPIKeyAuth,
+    litellm_changed_by: str | None,
+) -> tuple[str, ...]:
+    keys: Final = await _token_tx_db(tx).find_many(where=_in_filter("user_id", user_ids))
+    if keys:
+        await _persist_deleted_verification_tokens(
+            keys=keys,  # pyright: ignore[reportArgumentType]  # generated row model carries the same columns as LiteLLM_VerificationToken
+            prisma_client=prisma_client,
+            user_api_key_dict=user_api_key_dict,
+            litellm_changed_by=litellm_changed_by,
+            tx=tx,
+        )
+        await _token_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids))
+    await _invitation_tx_db(tx).delete_many(
+        where=_any_filter(
+            _in_filter("user_id", user_ids),
+            _in_filter("created_by", user_ids),
+            _in_filter("updated_by", user_ids),
+        )
+    )
+    await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids))
+    await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids))
+    await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids))
+    return tuple(k.token for k in keys)
+
+
+async def _delete_users_tx(
+    prisma_client: PrismaClient,
+    users: Sequence["prisma_models.LiteLLM_UserTable"],
+    teams_of: Mapping[str, frozenset[str]],
+    user_api_key_dict: UserAPIKeyAuth,
+    litellm_changed_by: str | None,
+) -> _UserBatchDeletion:
+    """Rewrites every team the users belong to and deletes their rows in one transaction, so a
+    failure anywhere rolls back the whole batch. Teams a user still names but which no longer exist
+    are skipped; the user row goes away regardless."""
+    async with prisma_client.tx(timeout=_BATCH_TX_TIMEOUT) as tx:
+        team_rows: Final = await _team_tx_db(tx).find_many(
+            where=_in_filter("team_id", frozenset(t for teams in teams_of.values() for t in teams))
+        )
+        team_ids: Final = tuple(sorted(t.team_id for t in team_rows))
+        removals: Final = MappingProxyType(
+            {
+                tid: await _remove_members_from_team(
+                    prisma_client,
+                    tx,
+                    tid,
+                    tuple(
+                        MemberDeleteRequest(user_id=u.user_id, user_email=u.user_email)
+                        for u in users
+                        if tid in teams_of[u.user_id]
+                    ),
+                    user_api_key_dict,
+                )
+                for tid in team_ids
+            }
+        )
+        deleted_key_tokens: Final = await _delete_user_rows(
+            prisma_client, tx, frozenset(u.user_id for u in users), user_api_key_dict, litellm_changed_by
+        )
+    return _UserBatchDeletion(
+        removals=removals,
+        deleted_key_tokens=deleted_key_tokens + tuple(t for r in removals.values() for t in r.deleted_key_tokens),
+    )
+
+
+async def _delete_users(
+    prisma_client: PrismaClient,
+    users: Sequence["prisma_models.LiteLLM_UserTable"],
+    teams_of: Mapping[str, frozenset[str]],
+    user_api_key_dict: UserAPIKeyAuth,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging | None,
+    litellm_proxy_admin_name: str | None,
+    litellm_changed_by: str | None,
+) -> _UserBatchDeletion | str:
+    """Returns the error message when the transaction rolled back, in which case no row was touched."""
+    user_ids: Final = frozenset(u.user_id for u in users)
+    try:
+        deletion: Final = await _delete_users_tx(prisma_client, users, teams_of, user_api_key_dict, litellm_changed_by)
+    except Exception as e:  # noqa: BLE001  # the rolled-back batch is reported per row, not as a request failure
+        verbose_proxy_logger.error("users/bulk_delete: failed to delete users %s: %s", sorted(user_ids), e)
+        return _error_message(e)
+    await delete_cache_key_objects(
+        hashed_tokens=deletion.deleted_key_tokens,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    await evict_and_broadcast(cache_keys=sorted(user_ids), user_api_key_cache=user_api_key_cache)
+    for removal in deletion.removals.values():
+        _emit_team_members_metric(removal.team)
+    audit_outcomes: Final = await _bounded(
+        UserManagementEventHooks.create_internal_user_audit_log(
+            user_id=u.user_id,
+            action="deleted",
+            litellm_changed_by=litellm_changed_by,
+            user_api_key_dict=user_api_key_dict,
+            litellm_proxy_admin_name=litellm_proxy_admin_name,
+            before_value=u.model_dump_json(exclude_none=True),
+        )
+        for u in users
+    )
+    for u, outcome in zip(users, audit_outcomes, strict=True):
+        if isinstance(outcome, BaseException):
+            verbose_proxy_logger.warning("Failed to create audit log for user %s: %s", u.user_id, outcome)
+    return deletion
+
+
+async def bulk_delete_users(
+    data: BulkDeleteUserRequest,
+    user_api_key_dict: UserAPIKeyAuth,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    proxy_logging_obj: ProxyLogging | None,
+    litellm_proxy_admin_name: str | None,
+    litellm_changed_by: str | None,
+) -> tuple[UserDeleteResult, ...]:
+    caller_is_proxy_admin: Final = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
+    caller_admin_org_ids: Final = await _caller_admin_org_ids(prisma_client, user_api_key_dict)
+    if not caller_is_proxy_admin and not caller_admin_org_ids:
+        raise _forbidden("Only PROXY_ADMIN or ORG_ADMIN users may delete users.")
+
+    unique_ids: Final = frozenset(data.user_ids)
+    rows: Final = await UserRepository(prisma_client).table.find_many(where=_in_filter("user_id", unique_ids))
+    rows_by_id: Final = MappingProxyType({row.user_id: row for row in rows})
+    target_memberships: Final = (
+        ()
+        if caller_is_proxy_admin
+        else await OrganizationMembershipRepository(prisma_client).table.find_many(
+            where=_in_filter("user_id", unique_ids)
+        )
+    )
+
+    def precheck_error(user_id: str) -> str | None:
+        if user_id not in rows_by_id:
+            return f"User id={user_id} not found"
+        if caller_is_proxy_admin:
+            return None
+        org_ids: Final = frozenset(
+            m.organization_id for m in target_memberships if m.user_id == user_id and m.organization_id
+        )
+        return _scope_error(user_id, org_ids, caller_admin_org_ids)
+
+    precheck_errors: Final = MappingProxyType({uid: precheck_error(uid) for uid in unique_ids})
+    candidates: Final = tuple(rows_by_id[uid] for uid in sorted(unique_ids) if precheck_errors[uid] is None)
+    candidate_ids: Final = frozenset(u.user_id for u in candidates)
+
+    memberships: Final = await TeamMembershipRepository(prisma_client).table.find_many(
+        where=_in_filter("user_id", candidate_ids)
+    )
+    teams_of: Final = MappingProxyType(
+        {
+            u.user_id: frozenset(u.teams) | frozenset(m.team_id for m in memberships if m.user_id == u.user_id)
+            for u in candidates
+        }
+    )
+    deletion: Final = (
+        await _delete_users(
+            prisma_client,
+            candidates,
+            teams_of,
+            user_api_key_dict,
+            user_api_key_cache,
+            proxy_logging_obj,
+            litellm_proxy_admin_name,
+            litellm_changed_by,
+        )
+        if candidates
+        else _UserBatchDeletion(removals=MappingProxyType({}), deleted_key_tokens=())
+    )
+
+    def result(index: int, user_id: str) -> UserDeleteResult:
+        if user_id in data.user_ids[:index]:
+            return UserDeleteResult(user_id=user_id, success=False, error=f"Duplicate user_id in request: {user_id}")
+        error: Final = precheck_errors[user_id]
+        if error is not None:
+            return UserDeleteResult(user_id=user_id, success=False, error=error)
+        if isinstance(deletion, str):
+            return UserDeleteResult(
+                user_id=user_id,
+                user_email=rows_by_id[user_id].user_email,
+                success=False,
+                error=f"Failed to delete user: {deletion}",
+            )
+        return UserDeleteResult(
+            user_id=user_id,
+            user_email=rows_by_id[user_id].user_email,
+            success=True,
+            teams_removed=tuple(tid for tid, r in deletion.removals.items() if user_id in r.removed),
+        )
+
+    return tuple(result(i, uid) for i, uid in enumerate(data.user_ids))

litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_ai_live_passthrough_logging_handler.py

@@ -5,18 +5,105 @@ Handles cost tracking and logging for Vertex AI Live API WebSocket passthrough e
 Supports different modalities: text, audio, video, and web search.
 """
 
+from collections.abc import Mapping, Sequence
 from datetime import datetime
-from typing import Any, Final
+from itertools import chain, pairwise
+from types import MappingProxyType
+from typing import Final, Literal, TypeAlias
 
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.vertex_ai.gemini.grounding_requests import GroundingRequests, calculate_grounding_requests
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import (
     BasePassthroughLoggingHandler,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import (
     PassThroughEndpointLoggingTypedDict,
 )
-from litellm.types.utils import LlmProviders, ModelResponse, Usage
-from litellm.utils import get_model_info
+from litellm.types.utils import (
+    CompletionTokensDetailsWrapper,
+    CostBreakdown,
+    LlmProviders,
+    ModelResponse,
+    PromptTokensDetailsWrapper,
+    Usage,
+)
+
+_NO_GROUNDING: Final = GroundingRequests(web_search_requests=None, google_maps_grounding_requests=None)
+
+_AGGREGATED_FIELDS: Final = frozenset(
+    {
+        "promptTokenCount",
+        "candidatesTokenCount",
+        "totalTokenCount",
+        "toolUsePromptTokenCount",
+        "promptTokensDetails",
+        "candidatesTokensDetails",
+    }
+)
+
+
+def _detail_entries(raw: object) -> tuple[Mapping[str, object], ...]:
+    """Narrow one turn's ``*TokensDetails`` value to the entries that are actually shaped like one."""
+    return tuple(entry for entry in raw if isinstance(entry, Mapping)) if isinstance(raw, Sequence) else ()
+
+
+def _grounding_metadata(websocket_messages: Sequence[object]) -> tuple[Mapping[str, object], ...]:
+    """Collect every ``serverContent.groundingMetadata`` a session emitted.
+
+    Live reports grounding in the server frames, never in ``usageMetadata``, so the per-query
+    charge has to be counted here rather than derived from the token totals.
+    """
+    return tuple(
+        metadata
+        for message in websocket_messages
+        if isinstance(message, Mapping)
+        for server_content in (message.get("serverContent"),)
+        if isinstance(server_content, Mapping)
+        for metadata in (server_content.get("groundingMetadata"),)
+        if isinstance(metadata, Mapping)
+    )
+
+
+def _turns(websocket_messages: Sequence[object]) -> tuple[tuple[object, ...], ...]:
+    """Split a session at every ``usageMetadata`` frame; frames after the last one never got their usage."""
+    closes: Final = tuple(
+        index + 1
+        for index, message in enumerate(websocket_messages)
+        if isinstance(message, Mapping) and isinstance(message.get("usageMetadata"), dict)
+    )
+    return tuple(tuple(websocket_messages[start:end]) for start, end in pairwise((0, *closes)))
+
+
+def _session_grounding_requests(websocket_messages: Sequence[object]) -> GroundingRequests:
+    per_turn: Final = tuple(
+        calculate_grounding_requests(_grounding_metadata(turn)) for turn in _turns(websocket_messages)
+    )
+    web_search_requests: Final = sum(requests.web_search_requests or 0 for requests in per_turn)
+    google_maps_grounding_requests: Final = sum(requests.google_maps_grounding_requests or 0 for requests in per_turn)
+    return GroundingRequests(
+        web_search_requests=web_search_requests or None,
+        google_maps_grounding_requests=google_maps_grounding_requests or None,
+    )
+
+
+_SummedField: TypeAlias = Literal[
+    "input_cost",
+    "output_cost",
+    "tool_usage_cost",
+    "cache_read_cost",
+    "cache_creation_cost",
+    "reasoning_cost",
+    "original_cost",
+    "discount_amount",
+    "margin_fixed_amount",
+    "margin_total_amount",
+]
+
+
+def _summed(breakdowns: Sequence[CostBreakdown], field: _SummedField) -> float | None:
+    values: Final = tuple(value for breakdown in breakdowns if (value := breakdown.get(field)) is not None)
+    return sum(values) if values else None
 
 
 class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler):
@@ -48,186 +135,110 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler):
         """Return the LLM provider name."""
         return LlmProviders.VERTEX_AI
 
+    @staticmethod
+    def _resolve_detail_counts(
+        details: Sequence[Mapping[str, object]],
+        declared_total: object,
+    ) -> tuple[tuple[str, int], ...]:
+        """
+        Pair each of one turn's ``*TokensDetails`` entries with its token count.
+
+        Live sometimes names the modality that carries the rest of a turn without a
+        ``tokenCount``, and reading the absent key as zero drops those tokens from the
+        breakdown, so real audio ends up priced as text. A lone unpriced entry therefore takes
+        whatever the turn's declared count leaves over. Two or more cannot be told apart, so
+        they are left out and the cost calculator charges the remainder as text.
+        """
+        priced: Final = tuple(
+            (str(detail.get("modality", "TEXT")), count)
+            for detail in details
+            if isinstance(count := detail.get("tokenCount"), int)
+        )
+        unpriced: Final = tuple(
+            str(detail.get("modality", "TEXT")) for detail in details if not isinstance(detail.get("tokenCount"), int)
+        )
+        if len(unpriced) != 1 or not isinstance(declared_total, int):
+            return priced
+        residual: Final = declared_total - sum(count for _, count in priced)
+        return priced if residual <= 0 else (*priced, (unpriced[0], residual))
+
+    @staticmethod
+    def _sum_by_modality(counts: Sequence[tuple[str, int]]) -> Mapping[str, int]:
+        """Total the (modality, tokenCount) pairs of one or more turns per modality."""
+        return MappingProxyType({modality: sum(c for m, c in counts if m == modality) for modality, _ in counts})
+
+    @staticmethod
+    def _merged_modality_totals(
+        snapshots: Sequence[Mapping[str, object]],
+        count_key: str,
+        details_key: str,
+    ) -> Mapping[str, int]:
+        """Total every turn's per-modality counts, so the breakdown adds up the way the totals do."""
+        return VertexAILivePassthroughLoggingHandler._sum_by_modality(
+            tuple(
+                chain.from_iterable(
+                    VertexAILivePassthroughLoggingHandler._resolve_detail_counts(
+                        _detail_entries(snapshot.get(details_key)), snapshot.get(count_key)
+                    )
+                    for snapshot in snapshots
+                )
+            )
+        )
+
     @staticmethod
     def _extract_usage_metadata_from_websocket_messages(
-        websocket_messages: list[dict],
+        websocket_messages: Sequence[object],
     ) -> dict | None:
         """
         Extract and aggregate usage metadata from a list of WebSocket messages.
 
+        Live emits one ``usageMetadata`` per turn and Google charges per turn for every token in
+        the session context window, which is the current turn's tokens plus all accumulated
+        tokens from previous turns, so the turns add up rather than restating each other. See
+        the Live API note under https://cloud.google.com/vertex-ai/generative-ai/pricing.
+
         Args:
             websocket_messages: List of WebSocket messages from the Live API
 
         Returns:
             Dictionary containing aggregated usage metadata, or None if not found
         """
-        all_usage_metadata: Final = []
+        snapshots: Final = tuple(
+            metadata
+            for message in websocket_messages
+            if isinstance(message, Mapping)
+            for metadata in (message.get("usageMetadata"),)
+            if isinstance(metadata, dict)
+        )
 
-        # Collect all usage metadata messages
-        for message in websocket_messages:
-            if isinstance(message, dict) and "usageMetadata" in message:
-                all_usage_metadata.append(message["usageMetadata"])
-
-        if not all_usage_metadata:
+        if not snapshots:
             return None
 
-        # If only one usage metadata, return it as-is
-        if len(all_usage_metadata) == 1:
-            return all_usage_metadata[0]
-
-        # Aggregate multiple usage metadata messages
-        aggregated: Final[dict[str, Any]] = {
-            "promptTokenCount": 0,
-            "candidatesTokenCount": 0,
-            "totalTokenCount": 0,
-            "promptTokensDetails": [],
-            "candidatesTokensDetails": [],
+        prompt_totals: Final = VertexAILivePassthroughLoggingHandler._merged_modality_totals(
+            snapshots, "promptTokenCount", "promptTokensDetails"
+        )
+        candidate_totals: Final = VertexAILivePassthroughLoggingHandler._merged_modality_totals(
+            snapshots, "candidatesTokenCount", "candidatesTokensDetails"
+        )
+        return {
+            **{key: value for key, value in snapshots[0].items() if key not in _AGGREGATED_FIELDS},
+            "promptTokenCount": sum(snapshot.get("promptTokenCount", 0) for snapshot in snapshots),
+            "candidatesTokenCount": sum(snapshot.get("candidatesTokenCount", 0) for snapshot in snapshots),
+            "totalTokenCount": sum(snapshot.get("totalTokenCount", 0) for snapshot in snapshots),
+            "toolUsePromptTokenCount": sum(snapshot.get("toolUsePromptTokenCount", 0) for snapshot in snapshots),
+            "promptTokensDetails": [
+                {"modality": modality, "tokenCount": count} for modality, count in prompt_totals.items() if count > 0
+            ],
+            "candidatesTokensDetails": [
+                {"modality": modality, "tokenCount": count} for modality, count in candidate_totals.items() if count > 0
+            ],
         }
 
-        # Aggregate token counts
-        for usage in all_usage_metadata:
-            aggregated["promptTokenCount"] += usage.get("promptTokenCount", 0)
-            aggregated["candidatesTokenCount"] += usage.get("candidatesTokenCount", 0)
-            aggregated["totalTokenCount"] += usage.get("totalTokenCount", 0)
-
-        # Aggregate token details by modality
-        modality_totals: Final = {}
-
-        for usage in all_usage_metadata:
-            # Process prompt tokens details
-            for detail in usage.get("promptTokensDetails", []):
-                modality = detail.get("modality", "TEXT")
-                token_count = detail.get("tokenCount", 0)
-
-                if modality not in modality_totals:
-                    modality_totals[modality] = {"prompt": 0, "candidate": 0}
-                modality_totals[modality]["prompt"] += token_count
-
-            # Process candidate tokens details
-            for detail in usage.get("candidatesTokensDetails", []):
-                modality = detail.get("modality", "TEXT")
-                token_count = detail.get("tokenCount", 0)
-
-                if modality not in modality_totals:
-                    modality_totals[modality] = {"prompt": 0, "candidate": 0}
-                modality_totals[modality]["candidate"] += token_count
-
-        # Convert aggregated modality totals back to details format
-        for modality, totals in modality_totals.items():
-            if totals["prompt"] > 0:
-                aggregated["promptTokensDetails"].append({"modality": modality, "tokenCount": totals["prompt"]})
-            if totals["candidate"] > 0:
-                aggregated["candidatesTokensDetails"].append({"modality": modality, "tokenCount": totals["candidate"]})
-
-        # Add any additional fields from the first usage metadata
-        first_usage: Final = all_usage_metadata[0]
-        for key, value in first_usage.items():
-            if key not in aggregated:
-                aggregated[key] = value
-
-        return aggregated
-
-    @staticmethod
-    def _calculate_live_api_cost(
-        model: str,
-        usage_metadata: dict,
-        custom_llm_provider: str = "vertex_ai",
-    ) -> float:
-        """
-        Calculate cost for Vertex AI Live API based on usage metadata.
-
-        Args:
-            model: The model name (e.g., "gemini-2.0-flash-live-preview-04-09")
-            usage_metadata: Usage metadata from the Live API response
-            custom_llm_provider: The LLM provider (default: "vertex_ai")
-
-        Returns:
-            Total cost in USD
-        """
-        try:
-            # Get model pricing information
-            model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
-
-            verbose_proxy_logger.debug("Vertex AI Live API model info for '%s': %s", model, model_info)
-
-            # Check if pricing info is available
-            if not model_info or not model_info.get("input_cost_per_token"):
-                verbose_proxy_logger.error("No pricing info found for %s in local model pricing database", model)
-                return 0.0
-
-            total_cost = 0.0
-
-            # Extract token counts from usage metadata
-            prompt_token_count: Final = usage_metadata.get("promptTokenCount", 0)
-            candidates_token_count: Final = usage_metadata.get("candidatesTokenCount", 0)
-
-            # Calculate base text token costs
-            input_cost_per_token: Final = model_info.get("input_cost_per_token", 0.0)
-            output_cost_per_token: Final = model_info.get("output_cost_per_token", 0.0)
-
-            total_cost += prompt_token_count * input_cost_per_token
-            total_cost += candidates_token_count * output_cost_per_token
-
-            # Handle modality-specific costs if present
-            prompt_tokens_details: Final = usage_metadata.get("promptTokensDetails", [])
-            candidates_tokens_details: Final = usage_metadata.get("candidatesTokensDetails", [])
-
-            # Process prompt tokens by modality
-            for detail in prompt_tokens_details:
-                modality = detail.get("modality", "TEXT")
-                token_count = detail.get("tokenCount", 0)
-
-                if modality == "AUDIO":
-                    audio_cost_per_token = model_info.get("input_cost_per_audio_token", 0.0)
-                    total_cost += token_count * audio_cost_per_token
-                elif modality == "VIDEO":
-                    # Video tokens are typically per second, but we'll treat as per token for now
-                    video_cost_per_token = model_info.get("input_cost_per_video_per_second", 0.0)
-                    total_cost += token_count * video_cost_per_token
-                # TEXT tokens are already handled above
-
-            # Process candidate tokens by modality
-            for detail in candidates_tokens_details:
-                modality = detail.get("modality", "TEXT")
-                token_count = detail.get("tokenCount", 0)
-
-                if modality == "AUDIO":
-                    audio_cost_per_token = model_info.get("output_cost_per_audio_token", 0.0)
-                    total_cost += token_count * audio_cost_per_token
-                elif modality == "VIDEO":
-                    # Video tokens are typically per second, but we'll treat as per token for now
-                    video_cost_per_token = model_info.get("output_cost_per_video_per_second", 0.0)
-                    total_cost += token_count * video_cost_per_token
-                # TEXT tokens are already handled above
-
-            # Handle web search costs if present
-            tool_use_prompt_token_count: Final = usage_metadata.get("toolUsePromptTokenCount", 0)
-            if tool_use_prompt_token_count > 0:
-                # Web search typically has a fixed cost per request
-                web_search_cost: Final = model_info.get("web_search_cost_per_request", 0.0)
-                if isinstance(web_search_cost, (int, float)) and web_search_cost > 0:
-                    total_cost += web_search_cost
-                else:
-                    # Fallback to token-based pricing for tool use
-                    total_cost += tool_use_prompt_token_count * input_cost_per_token
-
-            verbose_proxy_logger.debug(
-                f"Vertex AI Live API cost calculation - Model: {model}, "
-                f"Prompt tokens: {prompt_token_count}, "
-                f"Candidate tokens: {candidates_token_count}, "
-                f"Total cost: ${total_cost:.6f}"
-            )
-
-            return total_cost
-
-        except Exception as e:
-            verbose_proxy_logger.error("Error calculating Vertex AI Live API cost: %s", e)
-            return 0.0
-
     @staticmethod
     def _create_usage_object_from_metadata(
         usage_metadata: dict,
         model: str,
+        grounding_requests: GroundingRequests = _NO_GROUNDING,
     ) -> Usage:
         """
         Create a LiteLLM Usage object from Live API usage metadata.
@@ -235,48 +246,124 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler):
         Args:
             usage_metadata: Usage metadata from the Live API response
             model: The model name
+            grounding_requests: The Search and Maps grounding requests summed over the session's
+                turns, matching the per-turn charge
 
         Returns:
             LiteLLM Usage object
         """
-        prompt_tokens: Final = usage_metadata.get("promptTokenCount", 0)
-        completion_tokens: Final = usage_metadata.get("candidatesTokenCount", 0)
-        total_tokens: Final = usage_metadata.get("totalTokenCount", 0)
+        prompt_by_modality: Final = VertexAILivePassthroughLoggingHandler._sum_by_modality(
+            VertexAILivePassthroughLoggingHandler._resolve_detail_counts(
+                _detail_entries(usage_metadata.get("promptTokensDetails")), usage_metadata.get("promptTokenCount")
+            )
+        )
+        candidates_by_modality: Final = VertexAILivePassthroughLoggingHandler._sum_by_modality(
+            VertexAILivePassthroughLoggingHandler._resolve_detail_counts(
+                _detail_entries(usage_metadata.get("candidatesTokensDetails")),
+                usage_metadata.get("candidatesTokenCount"),
+            )
+        )
 
-        # Create modality-specific token details if available
-        prompt_tokens_details: Final = usage_metadata.get("promptTokensDetails", [])
-        candidates_tokens_details: Final = usage_metadata.get("candidatesTokensDetails", [])
-
-        # Extract text tokens from details
-        text_prompt_tokens = 0
-        text_completion_tokens = 0
-
-        for detail in prompt_tokens_details:
-            if detail.get("modality") == "TEXT":
-                text_prompt_tokens = detail.get("tokenCount", 0)
-                break
-
-        for detail in candidates_tokens_details:
-            if detail.get("modality") == "TEXT":
-                text_completion_tokens = detail.get("tokenCount", 0)
-                break
-
-        # If no text tokens found in details, use total counts
-        if text_prompt_tokens == 0:
-            text_prompt_tokens = prompt_tokens
-        if text_completion_tokens == 0:
-            text_completion_tokens = completion_tokens
+        prompt_tokens: Final = usage_metadata.get("promptTokenCount", 0) or sum(prompt_by_modality.values())
+        completion_tokens: Final = usage_metadata.get("candidatesTokenCount", 0) or sum(candidates_by_modality.values())
 
         return Usage(
-            prompt_tokens=text_prompt_tokens,
-            completion_tokens=text_completion_tokens,
-            total_tokens=total_tokens,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=usage_metadata.get("totalTokenCount", 0) or (prompt_tokens + completion_tokens),
+            prompt_tokens_details=PromptTokensDetailsWrapper(
+                text_tokens=prompt_by_modality.get("TEXT"),
+                audio_tokens=prompt_by_modality.get("AUDIO"),
+                image_tokens=prompt_by_modality.get("IMAGE"),
+                video_tokens=prompt_by_modality.get("VIDEO"),
+                tool_use_tokens=usage_metadata.get("toolUsePromptTokenCount") or None,
+                web_search_requests=grounding_requests.web_search_requests,
+                google_maps_grounding_requests=grounding_requests.google_maps_grounding_requests,
+            ),
+            completion_tokens_details=CompletionTokensDetailsWrapper(
+                text_tokens=candidates_by_modality.get("TEXT"),
+                audio_tokens=candidates_by_modality.get("AUDIO"),
+                image_tokens=candidates_by_modality.get("IMAGE"),
+                video_tokens=candidates_by_modality.get("VIDEO"),
+            ),
         )
 
+    def _session_usage(self, websocket_messages: Sequence[object], model: str) -> Usage | None:
+        usage_metadata: Final = self._extract_usage_metadata_from_websocket_messages(websocket_messages)
+        if usage_metadata is None:
+            return None
+        return self._create_usage_object_from_metadata(
+            usage_metadata=usage_metadata,
+            grounding_requests=_session_grounding_requests(websocket_messages),
+            model=model,
+        )
+
+    def _turn_cost(
+        self,
+        turn: Sequence[object],
+        model: str,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> tuple[float, CostBreakdown] | None:
+        usage: Final = self._session_usage(turn, model)
+        if usage is None:
+            return None
+        cost: Final = logging_obj._response_cost_calculator(  # pyright: ignore[reportPrivateUsage]  # the call's own calculator keeps custom pricing and the deployment's region in step with the spend row
+            result=ModelResponse(model=model, usage=usage),
+            litellm_model_name=model,
+        )
+        if cost is None:
+            return None
+        breakdown: Final = logging_obj.cost_breakdown
+        return None if breakdown is None else (cost, breakdown)
+
+    def _session_cost(
+        self,
+        websocket_messages: Sequence[object],
+        model: str,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> float | None:
+        """Price each turn on its own tokens and grounding, so two grounded turns pay the query fee twice.
+
+        The fixed cost margin is a flat per-request fee, so the session's single spend row carries it once
+        rather than once per turn.
+        """
+        turn_costs: Final = tuple(self._turn_cost(turn, model, logging_obj) for turn in _turns(websocket_messages))
+        priced: Final = tuple(turn_cost for turn_cost in turn_costs if turn_cost is not None)
+        if not priced or len(priced) != len(turn_costs):
+            return None
+        breakdowns: Final = tuple(breakdown for _, breakdown in priced)
+        first: Final = breakdowns[0]
+        fixed_margin: Final = first.get("margin_fixed_amount") or 0.0
+        duplicated_fixed_margin: Final = fixed_margin * (len(priced) - 1)
+        total_cost: Final = sum(cost for cost, _ in priced) - duplicated_fixed_margin
+        summed_margin_total: Final = _summed(breakdowns, "margin_total_amount")
+        margin_total_amount: Final = (
+            None if summed_margin_total is None else summed_margin_total - duplicated_fixed_margin
+        )
+        logging_obj.set_cost_breakdown(
+            input_cost=_summed(breakdowns, "input_cost") or 0.0,
+            output_cost=_summed(breakdowns, "output_cost") or 0.0,
+            total_cost=total_cost,
+            cost_for_built_in_tools_cost_usd_dollar=_summed(breakdowns, "tool_usage_cost") or 0.0,
+            original_cost=_summed(breakdowns, "original_cost"),
+            discount_percent=first.get("discount_percent"),
+            discount_amount=_summed(breakdowns, "discount_amount"),
+            margin_percent=first.get("margin_percent"),
+            margin_fixed_amount=first.get("margin_fixed_amount"),
+            margin_total_amount=margin_total_amount,
+            cache_read_cost=_summed(breakdowns, "cache_read_cost"),
+            cache_creation_cost=_summed(breakdowns, "cache_creation_cost"),
+            reasoning_cost=_summed(breakdowns, "reasoning_cost"),
+            service_tier=first.get("service_tier"),
+            data_residency=first.get("data_residency"),
+            vertex_location=first.get("vertex_location"),
+        )
+        return total_cost
+
     def vertex_ai_live_passthrough_handler(
         self,
-        websocket_messages: list[dict],
-        logging_obj,
+        websocket_messages: Sequence[object],
+        logging_obj: LiteLLMLoggingObj,
         url_route: str,
         start_time: datetime,
         end_time: datetime,
@@ -300,34 +387,25 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler):
         """
         try:
             # Extract model from request body or kwargs
-            model: Final = kwargs.get("model", "gemini-2.0-flash-live-preview-04-09")
+            requested_model: Final = kwargs.get("model")
+            model: Final = (
+                requested_model if isinstance(requested_model, str) else "gemini-2.0-flash-live-preview-04-09"
+            )
             custom_llm_provider: Final = kwargs.get("custom_llm_provider", "vertex_ai")
             verbose_proxy_logger.debug(
                 "Vertex AI Live API model: %s, custom_llm_provider: %s", model, custom_llm_provider
             )
 
-            # Extract usage metadata from WebSocket messages
-            usage_metadata: Final = self._extract_usage_metadata_from_websocket_messages(websocket_messages)
+            usage: Final = self._session_usage(websocket_messages, model)
 
-            if not usage_metadata:
+            if usage is None:
                 verbose_proxy_logger.warning("No usage metadata found in Vertex AI Live API WebSocket messages")
                 return {
                     "result": None,
                     "kwargs": kwargs,
                 }
 
-            # Calculate cost using Live API specific pricing
-            response_cost: Final = self._calculate_live_api_cost(
-                model=model,
-                usage_metadata=usage_metadata,
-                custom_llm_provider=custom_llm_provider,
-            )
-
-            # Create Usage object for standard LiteLLM logging
-            usage: Final = self._create_usage_object_from_metadata(
-                usage_metadata=usage_metadata,
-                model=model,
-            )
+            response_cost: Final = self._session_cost(websocket_messages, model, logging_obj)
 
             # Create a mock ModelResponse for standard logging
             litellm_model_response: Final = ModelResponse(
@@ -338,9 +416,9 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler):
                 usage=usage,
                 choices=[],
             )
+            if response_cost is not None:
+                litellm_model_response._hidden_params["response_cost"] = response_cost  # pyright: ignore[reportPrivateUsage]  # the logger reads the cost off the response's hidden params; the constructor's hidden_params kwarg is reset by pydantic
 
-            # Update kwargs with cost information
-            kwargs["response_cost"] = response_cost
             kwargs["model"] = model
             kwargs["custom_llm_provider"] = custom_llm_provider
 
@@ -348,12 +426,15 @@ class VertexAILivePassthroughLoggingHandler(BasePassthroughLoggingHandler):
             import re
 
             allowed_pattern: Final = re.compile(r"^[A-Za-z0-9._\-:]+$")
-            safe_model: Final = model if isinstance(model, str) and allowed_pattern.match(model) else "[REDACTED]"
+            safe_model: Final = model if allowed_pattern.match(model) else "[REDACTED]"
             verbose_proxy_logger.debug(
-                f"Vertex AI Live API passthrough cost tracking - "
-                f"Model: {safe_model}, Cost: ${response_cost:.6f}, "
-                f"Prompt tokens: {usage.prompt_tokens}, "
-                f"Completion tokens: {usage.completion_tokens}"
+                "Vertex AI Live API passthrough cost tracking - Model: %s, "
+                "Prompt tokens: %s %s, Completion tokens: %s %s",
+                safe_model,
+                usage.prompt_tokens,
+                usage.prompt_tokens_details,
+                usage.completion_tokens,
+                usage.completion_tokens_details,
             )
 
             return {

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -2090,6 +2090,22 @@ def _rewrite_vertex_live_setup_model(text_data: str, setup_model_rewriter: Calla
     return json.dumps({**message, "setup": {**setup, "model": rewritten_model}})  # mutable-ok: one-shot json payload
 
 
+def _resolved_vertex_live_setup(
+    setup_data: Mapping[str, object], setup_model_rewriter: Callable[[str], str] | None
+) -> Mapping[str, object]:
+    """
+    Give the model extractor the same fully qualified path the upstream will receive.
+
+    Clients may name a bare gateway alias, which the rewriter turns into a ``projects/...`` path before
+    it reaches Vertex. The extractor only reads a path containing ``/models/``, so running it on the raw
+    frame logs the session as ``unknown`` at no cost, which is precisely the supported client form
+    """
+    setup_model: Final = setup_data.get("model")
+    if setup_model_rewriter is None or not isinstance(setup_model, str):
+        return setup_data
+    return {**setup_data, "model": setup_model_rewriter(setup_model)}
+
+
 def _truncated_close_reason(reason: str) -> str:
     """
     Fit a close reason inside the byte budget a WebSocket close frame allows, without splitting a character
@@ -2314,7 +2330,9 @@ async def websocket_passthrough_request(
                                             setup_data,
                                         )
                                         if isinstance(setup_data, dict) and "model" in setup_data:
-                                            extracted_model = _extract_model_from_vertex_ai_setup(setup_data)
+                                            extracted_model = _extract_model_from_vertex_ai_setup(
+                                                _resolved_vertex_live_setup(setup_data, setup_model_rewriter)
+                                            )
                                             if extracted_model:
                                                 kwargs["model"] = extracted_model
                                                 kwargs["custom_llm_provider"] = "vertex_ai-language-models"

litellm/proxy/policy_engine/pipeline_executor.py

@@ -58,15 +58,6 @@ class UndeliverableStreamRewrite(Exception):
         self.guardrail_name: Final = guardrail_name
 
 
-class UnappliableRequestRewrite(Exception):
-    def __init__(self, guardrail_name: str) -> None:
-        super().__init__(
-            f"Guardrail '{guardrail_name}' rewrote the request in a way this endpoint cannot apply, "
-            "so the request was rejected rather than sent unrewritten"
-        )
-        self.guardrail_name: Final = guardrail_name
-
-
 def _tool_call_shape(tool_call: object) -> tuple[object, object]:
     plain: Final = tool_call.model_dump() if isinstance(tool_call, BaseModel) else tool_call
     function: Final = plain.get("function") if isinstance(plain, Mapping) else None

litellm/proxy/proxy_server.py

@@ -476,9 +476,10 @@ from litellm.proxy.hooks.prompt_injection_detection import (
 from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger, run_spend_event
 from litellm.proxy.image_endpoints.endpoints import router as image_router
 from litellm.proxy.list_api.common import (
-    PROBLEM_TYPE_BASE,
     ManagementProblem,
+    ValidationErrorDetail,
     problem_response,
+    request_validation_problem,
 )
 from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
 from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
@@ -601,7 +602,6 @@ from litellm.proxy.spend_tracking.spend_event_producer import (
     SpendEventProducer,
     build_spend_event_producer,
 )
-from litellm.types.proxy.management_endpoints.management_v1 import ProblemDetail
 
 try:
     from litellm.proxy.enterprise_billing.billing_metrics import (
@@ -928,6 +928,7 @@ def cleanup_router_config_variables():
         user_custom_auth_path, \
         user_custom_key_generate, \
         user_custom_key_update, \
+        user_custom_key_policy, \
         user_custom_sso, \
         user_custom_ui_sso_sign_in_handler, \
         use_background_health_checks, \
@@ -945,6 +946,7 @@ def cleanup_router_config_variables():
     user_custom_auth_path = None
     user_custom_key_generate = None
     user_custom_key_update = None
+    user_custom_key_policy = None
     TEAM_METADATA_VALIDATOR_REGISTRY.set(None)
     TEAM_METADATA_SCHEMA_REGISTRY.set(())
     user_custom_sso = None
@@ -1787,27 +1789,13 @@ class _ExceptionRow(TypedDict, total=False):
     exception_counts: Mapping[str, int]
 
 
-class _ValidationErrorDetail(TypedDict):
-    loc: tuple[int | str, ...]
-    msg: str
-
-
 @app.exception_handler(RequestValidationError)
 async def otel_request_validation_exception_handler(request: Request, exc: RequestValidationError):
     if request.url.path.startswith(MANAGEMENT_V1_PREFIX):
-        _close_dangling_otel_server_span(request, 400, exc=exc)
-        validation_errors: Final[Sequence[_ValidationErrorDetail]] = exc.errors()
-        return problem_response(
-            ProblemDetail(
-                type=f"{PROBLEM_TYPE_BASE}invalid-query-parameter",
-                title="Invalid query parameter",
-                status=400,
-                detail="; ".join(
-                    f"{'.'.join(str(part) for part in error['loc'][1:])}: {error['msg']}" for error in validation_errors
-                )
-                or "The request query parameters are invalid.",
-            )
-        )
+        validation_errors: Final[Sequence[ValidationErrorDetail]] = exc.errors()
+        problem: Final = request_validation_problem(validation_errors)
+        _close_dangling_otel_server_span(request, problem.status, exc=exc)
+        return problem_response(problem)
     _close_dangling_otel_server_span(request, 422, exc=exc)
     return JSONResponse(
         status_code=422,
@@ -2369,6 +2357,7 @@ user_custom_key_generate = None
 _pkce_no_redis_warning_emitted: bool = False
 _cp_no_redis_warning_emitted: bool = False
 user_custom_key_update = None
+user_custom_key_policy = None
 user_custom_sso = None
 user_custom_ui_sso_sign_in_handler = None
 use_background_health_checks = None
@@ -4256,6 +4245,7 @@ _DB_OVERLAY_REMOTE_MODULE_STR_FIELDS: Final[dict[str, tuple[str, ...]]] = {
         "custom_auth",
         "custom_key_generate",
         "custom_key_update",
+        "custom_key_policy",
         "custom_team_metadata_validate",
         "custom_sso",
         "custom_ui_sso_sign_in_handler",
@@ -5405,6 +5395,7 @@ class ProxyConfig:
             user_custom_auth_path, \
             user_custom_key_generate, \
             user_custom_key_update, \
+            user_custom_key_policy, \
             user_custom_sso, \
             user_custom_ui_sso_sign_in_handler, \
             use_background_health_checks, \
@@ -5942,6 +5933,10 @@ class ProxyConfig:
             if custom_key_update is not None:
                 user_custom_key_update = get_instance_fn(value=custom_key_update, config_file_path=config_file_path)
 
+            custom_key_policy: Final = general_settings.get("custom_key_policy", None)
+            if custom_key_policy is not None:
+                user_custom_key_policy = get_instance_fn(value=custom_key_policy, config_file_path=config_file_path)
+
             custom_team_metadata_validate: Final = general_settings.get("custom_team_metadata_validate", None)
             TEAM_METADATA_VALIDATOR_REGISTRY.set(
                 get_instance_fn(value=custom_team_metadata_validate, config_file_path=config_file_path)
@@ -9546,6 +9541,7 @@ class ProxyStartupEvent:
         gate the first duration window.
         """
         await generate_key_helper_fn(
+            llm_router=llm_router,
             request_type="user",
             table_name="user",
             user_id=LITELLM_PROXY_BUDGET_NAME,
@@ -16290,6 +16286,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str:
     global master_key, general_settings
 
     response: Final = await generate_key_helper_fn(
+        llm_router=llm_router,
         request_type="key",
         **{
             "user_role": user_obj.user_role,

litellm/proxy/utils.py

@@ -3793,6 +3793,7 @@ def jsonify_object(data: dict) -> dict:
 # Bounded to prevent memory leaks from accumulated rotations.
 _deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(max_size=1000)
 _DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60
+_PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5)
 
 
 async def _lookup_deprecated_key(
@@ -4171,13 +4172,13 @@ class PrismaClient:
             return self.db.read_target
         return self.db
 
-    def tx(self) -> "TransactionManager":
+    def tx(self, *, timeout: timedelta = _PRISMA_DEFAULT_TX_TIMEOUT) -> "TransactionManager":
         """Open an interactive transaction on the writer.
 
         Callers go through this instead of reaching into ``self.db`` so writer
         selection and read-replica routing stay encapsulated in the wrapper.
         """
-        return cast("TransactionManager", self.db.tx())  # cast-ok: wrappers delegate tx via __getattr__ (untyped)
+        return cast("TransactionManager", self.db.tx(timeout=timeout))  # cast-ok: untyped __getattr__ delegate
 
     def get_request_status(self, payload: dict | SpendLogsPayload) -> Literal["success", "failure"]:
         """

litellm/realtime_api/main.py

@@ -30,6 +30,7 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import CallTypes, LlmProviders
 from litellm.utils import ProviderConfigManager
 
+from ..litellm_core_utils.credential_accessor import CredentialAccessor
 from ..litellm_core_utils.get_litellm_params import get_litellm_params
 from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from ..llms.azure.common_utils import get_azure_ad_token
@@ -54,6 +55,17 @@ xai_realtime: Final = XAIRealtime()
 vertex_llm_base: Final = VertexBase()
 base_llm_http_handler = BaseLLMHTTPHandler()
 _EMPTY_MODEL_PARAMS: Final[Mapping[str, Any]] = MappingProxyType({})
+_EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
+
+
+def _model_params_with_stored_credentials(model_params: Mapping[str, Any]) -> Mapping[str, Any]:
+    credential_name: Final = model_params.get("litellm_credential_name")
+    credential_values: Final = (
+        CredentialAccessor.get_credential_values(credential_name)
+        if isinstance(credential_name, str)
+        else _EMPTY_MODEL_PARAMS
+    )
+    return MappingProxyType({**credential_values, **model_params})
 
 
 def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]:
@@ -591,13 +603,15 @@ def _azure_realtime_health_protocol(
 
 def _realtime_health_check_auth_headers(
     custom_llm_provider: str, api_key: str | None, model_params: Mapping[str, Any]
-) -> Mapping[str, str | None]:
-    if custom_llm_provider != "azure":
-        return MappingProxyType({"api-key": api_key})
-    return azure_realtime.get_auth_headers(
-        api_key=api_key,
-        azure_ad_token=(None if api_key else get_azure_ad_token(GenericLiteLLMParams(**model_params))),
-    )
+) -> Mapping[str, str]:
+    if custom_llm_provider == "azure":
+        return azure_realtime.get_auth_headers(
+            api_key=api_key,
+            azure_ad_token=(None if api_key else get_azure_ad_token(GenericLiteLLMParams(**model_params))),
+        )
+    if api_key is None:
+        return _EMPTY_AUTH_HEADERS
+    return MappingProxyType({"Authorization": f"Bearer {api_key}"})
 
 
 async def _realtime_health_check(
@@ -629,34 +643,46 @@ async def _realtime_health_check(
     """
     import websockets
 
+    resolved_params: Final = _model_params_with_stored_credentials(model_params or _EMPTY_MODEL_PARAMS)
+    resolved_api_key: Final = cast(  # cast-ok: provider parameters expose optional string credentials
+        str | None, api_key or resolved_params.get("api_key")
+    )
+    resolved_api_base: Final = cast(  # cast-ok: provider parameters expose optional string endpoints
+        str | None, api_base or resolved_params.get("api_base")
+    )
+    resolved_api_version: Final = cast(  # cast-ok: provider parameters expose optional string versions
+        str | None, api_version or resolved_params.get("api_version")
+    )
     url: str | None = None
     auth_headers: Final = _realtime_health_check_auth_headers(
         custom_llm_provider=custom_llm_provider,
-        api_key=api_key,
-        model_params=model_params or _EMPTY_MODEL_PARAMS,
+        api_key=resolved_api_key,
+        model_params=resolved_params,
     )
     if custom_llm_provider == "azure":
         resolved_protocol, azure_query_params = _azure_realtime_health_protocol(
             model=model,
             realtime_protocol=realtime_protocol,
-            model_params=model_params or _EMPTY_MODEL_PARAMS,
+            model_params=resolved_params,
         )
         url = azure_realtime._construct_url(
-            api_base=api_base or "",
+            api_base=resolved_api_base or "",
             model=model,
-            api_version=api_version or "2024-10-01-preview",
+            api_version=resolved_api_version or "2024-10-01-preview",
             realtime_protocol=resolved_protocol,
             query_params=azure_query_params,
         )
     elif custom_llm_provider == "openai":
         url = openai_realtime._construct_url(
-            api_base=api_base or "https://api.openai.com/",
+            api_base=resolved_api_base or "https://api.openai.com/",
             query_params={"model": model},
         )
     elif custom_llm_provider == "xai":
-        url = xai_realtime._construct_url(api_base=api_base or "https://api.x.ai/v1", query_params={"model": model})
+        url = xai_realtime._construct_url(
+            api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model}
+        )
     elif custom_llm_provider == "vertex_ai":
-        vertex_model_params: Final = model_params or {}
+        vertex_model_params: Final = dict(resolved_params)
         resolved_location: Final = vertex_llm_base.get_vertex_region(
             vertex_region=VertexBase.safe_get_vertex_ai_location(vertex_model_params),
             model=model,
@@ -675,19 +701,19 @@ async def _realtime_health_check(
             project=resolved_project,
             location=resolved_location,
         )
-        url = vertex_realtime_config.get_complete_url(api_base=api_base, model=model)
-        ssl_context = get_shared_realtime_ssl_context()
+        url = vertex_realtime_config.get_complete_url(api_base=resolved_api_base, model=model)
+        vertex_ssl_context: Final = get_shared_realtime_ssl_context()
         headers: Final = vertex_realtime_config.validate_environment(headers={}, model=model, api_key=None)
         async with websockets.connect(
             url,
             additional_headers=headers,
             max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
-            ssl=ssl_context,
+            ssl=vertex_ssl_context,
         ):
             return True
     else:
         raise ValueError(f"Unsupported model: {model}")
-    ssl_context = get_shared_realtime_ssl_context()
+    ssl_context: Final = get_shared_realtime_ssl_context()
     async with websockets.connect(
         url,
         additional_headers=auth_headers,

litellm/repositories/base_repository.py

@@ -117,3 +117,13 @@ class BaseRepository(ABC, Generic[T]):
         """Check if a record exists."""
         record: Final = await self.table.find_unique(where={id_field: id_value})
         return record is not None
+
+
+def is_unique_violation(exc: BaseException) -> bool:
+    try:
+        from prisma.errors import UniqueViolationError
+    except ImportError:
+        return "P2002" in str(exc) or "unique constraint" in str(exc).lower()
+    if isinstance(exc, UniqueViolationError):
+        return True
+    return getattr(exc, "code", None) == "P2002"

litellm/repositories/prisma_protocols.py

@@ -12,6 +12,11 @@ from typing import Protocol, TypeVar
 RowT_co = TypeVar("RowT_co", covariant=True)
 
 
+class DatabaseClient(Protocol):
+    @property
+    def db(self) -> object: ...
+
+
 class TableActions(Protocol[RowT_co]):
     """The prisma-client-py per-model action surface, keyed to the row it returns.
 

litellm/responses/additional_tools.py

@@ -0,0 +1,65 @@
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Final, cast  # noqa: TID251  # validating the openai tool union strips vendor keys from raw tools
+
+from pydantic import BaseModel, ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam
+
+ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
+
+
+class _InputItemType(BaseModel):
+    type: str = ""
+
+
+class _AdditionalToolsItem(BaseModel):
+    tools: tuple[dict[str, object], ...] = ()
+
+
+@dataclass(frozen=True, slots=True)
+class HoistedAdditionalTools:
+    input: str | ResponseInputParam
+    tools: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
+    hoisted: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]
+
+
+def _is_additional_tools_item(item: object) -> bool:
+    try:
+        return _InputItemType.model_validate(item).type == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE
+    except ValidationError:
+        return False
+
+
+def _tools_of_item(item: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]:
+    try:
+        parsed: Final = _AdditionalToolsItem.model_validate(item)
+    except ValidationError:
+        return ()
+    return tuple(
+        cast(
+            "ALL_RESPONSES_API_TOOL_PARAMS", tool
+        )  # cast-ok: nested tools carry the same raw tool JSON as top-level tools
+        for tool in parsed.tools
+    )
+
+
+def hoist_additional_tools(
+    input: str | ResponseInputParam,
+    tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None,
+) -> HoistedAdditionalTools:
+    existing: Final = tuple(tools or ())
+    if isinstance(input, str):
+        return HoistedAdditionalTools(input=input, tools=existing, hoisted=())
+    items: Final = tuple(item for item in input if _is_additional_tools_item(item))
+    if not items:
+        return HoistedAdditionalTools(input=input, tools=existing, hoisted=())
+    hoisted: Final = tuple(tool for item in items for tool in _tools_of_item(item))
+    verbose_logger.debug(
+        "Responses API: hoisting %d tool(s) out of %d 'additional_tools' input item(s) into the top-level tools param.",
+        len(hoisted),
+        len(items),
+    )
+    remaining_input: Final = [item for item in input if not _is_additional_tools_item(item)]
+    return HoistedAdditionalTools(input=remaining_input, tools=(*existing, *hoisted), hoisted=hoisted)

litellm/responses/litellm_completion_transformation/custom_tools.py

@@ -39,15 +39,38 @@ def openai_shaped_tool_call_item_id(item_type: str, tool_id: str) -> str:
     return f"{prefix}_{tool_id}"
 
 
+class _ToolNameFields(BaseModel):
+    type: str = ""
+    name: str = ""
+    tools: tuple[object, ...] = ()
+
+
+def _tool_name_fields_of(tool: object) -> _ToolNameFields | None:
+    try:
+        return _ToolNameFields.model_validate(tool)
+    except ValidationError:
+        return None
+
+
+def _custom_tool_name_of(tool: object) -> str | None:
+    parsed: Final = _tool_name_fields_of(tool)
+    if parsed is None or parsed.type != "custom" or not parsed.name:
+        return None
+    return parsed.name
+
+
+def _nested_tools_of(tool: object) -> tuple[object, ...]:
+    parsed: Final = _tool_name_fields_of(tool)
+    if parsed is None or parsed.type != "namespace":
+        return ()
+    return parsed.tools
+
+
 def extract_custom_tool_names(tools: Sequence[object] | None) -> set[str]:
-    """Extract names of tools originally defined as ``type: "custom"``."""
-    if not tools:
-        return set()
-    names: Final[set[str]] = set()
-    for tool in tools:
-        if isinstance(tool, dict) and tool.get("type") == "custom" and "name" in tool:
-            names.add(tool["name"])
-    return names
+    """Extract names of ``type: "custom"`` tools, at the top level or one level inside a ``namespace`` tool."""
+    top_level: Final = tuple(tools or ())
+    nested: Final = tuple(nested_tool for tool in top_level for nested_tool in _nested_tools_of(tool))
+    return {name for tool in (*top_level, *nested) if (name := _custom_tool_name_of(tool)) is not None}
 
 
 def is_custom_tool_call(tool_name: str, custom_tool_names: set[str]) -> bool:
@@ -143,7 +166,7 @@ def validated_allowed_callers(value: object) -> list[str] | None:
         raise ValueError("allowed_callers must be a list of strings") from exc
 
 
-def _grammar_suffix(fmt: object) -> str:
+def custom_tool_grammar_suffix(fmt: object) -> str:
     try:
         parsed: Final = _CustomToolFormat.model_validate(fmt)
     except ValidationError:
@@ -167,7 +190,9 @@ def convert_custom_tool_to_function_tool(tool: Mapping[str, object]) -> ChatComp
     raw_name: Final = tool.get("name")
     name: Final = raw_name if isinstance(raw_name, str) else ""
     raw_description: Final = tool.get("description")
-    description = (raw_description if isinstance(raw_description, str) else "") + _grammar_suffix(tool.get("format"))
+    description: Final = (raw_description if isinstance(raw_description, str) else "") + custom_tool_grammar_suffix(
+        tool.get("format")
+    )
     allowed_callers: Final = validated_allowed_callers(tool.get("allowed_callers"))
     function_chunk: Final = ChatCompletionToolParamFunctionChunk(
         name=name,

litellm/responses/litellm_completion_transformation/handler.py

@@ -6,6 +6,7 @@ from collections.abc import Coroutine, Mapping
 from typing import Final
 
 import litellm
+from litellm.responses.additional_tools import hoist_additional_tools
 from litellm.responses.litellm_completion_transformation.streaming_iterator import (
     LiteLLMCompletionStreamingIterator,
 )
@@ -37,11 +38,16 @@ class LiteLLMCompletionTransformationHandler:
         | BaseResponsesAPIStreamingIterator
         | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
     ):
+        hoisted: Final = hoist_additional_tools(input, responses_api_request.get("tools"))
+        bridged_input: Final = hoisted.input
+        bridged_request: Final[ResponsesAPIOptionalRequestParams] = (
+            {**responses_api_request, "tools": list(hoisted.tools)} if hoisted.hoisted else responses_api_request
+        )
         litellm_completion_request: Final[dict] = (
             LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
                 model=model,
-                input=input,
-                responses_api_request=responses_api_request,
+                input=bridged_input,
+                responses_api_request=bridged_request,
                 custom_llm_provider=custom_llm_provider,
                 stream=stream,
                 extra_headers=extra_headers,
@@ -52,8 +58,8 @@ class LiteLLMCompletionTransformationHandler:
         if _is_async:
             return self.async_response_api_handler(
                 litellm_completion_request=litellm_completion_request,
-                request_input=input,
-                responses_api_request=responses_api_request,
+                request_input=bridged_input,
+                responses_api_request=bridged_request,
                 **kwargs,
             )
 
@@ -70,8 +76,8 @@ class LiteLLMCompletionTransformationHandler:
             responses_api_response: Final[ResponsesAPIResponse] = (
                 LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response(
                     chat_completion_response=litellm_completion_response,
-                    request_input=input,
-                    responses_api_request=responses_api_request,
+                    request_input=bridged_input,
+                    responses_api_request=bridged_request,
                 )
             )
 
@@ -81,8 +87,8 @@ class LiteLLMCompletionTransformationHandler:
             return LiteLLMCompletionStreamingIterator(
                 model=model,
                 litellm_custom_stream_wrapper=litellm_completion_response,
-                request_input=input,
-                responses_api_request=responses_api_request,
+                request_input=bridged_input,
+                responses_api_request=bridged_request,
                 custom_llm_provider=custom_llm_provider,
                 litellm_metadata=kwargs.get("litellm_metadata", {}),
             )

litellm/responses/litellm_completion_transformation/streaming_iterator.py

@@ -8,6 +8,7 @@ from litellm.main import stream_chunk_builder
 from litellm.responses.litellm_completion_transformation.custom_tools import (
     build_tool_call_item_kwargs,
     extract_custom_tool_names,
+    is_custom_tool_call,
     serialize_tool_call_arguments,
 )
 from litellm.responses.litellm_completion_transformation.transformation import (
@@ -166,6 +167,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             return tool_name, namespace
         return fn_name, None
 
+    def _tool_call_item_kwargs(self, call_id: str, fn_name: str, arguments: str, status: str) -> dict[str, str]:
+        item_kwargs: Final = build_tool_call_item_kwargs(call_id, fn_name, arguments, status, self._custom_tool_names)
+        if is_custom_tool_call(fn_name, self._custom_tool_names):
+            return item_kwargs
+        tool_name, tool_namespace = self._responses_namespace_tool_call_fields(fn_name)
+        namespace_kwargs: Final = {"namespace": tool_namespace} if tool_namespace else {}
+        return {**item_kwargs, "name": tool_name, **namespace_kwargs}
+
     def _is_reasoning_end(self, chunk):
         delta: Final = chunk.choices[0].delta
 
@@ -244,17 +253,13 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             else:
                 fn_name = str(getattr(fn, "name", "") or "")
                 fn_args_delta = serialize_tool_call_arguments(getattr(fn, "arguments", ""))
-            tool_name, tool_namespace = self._responses_namespace_tool_call_fields(fn_name)
             output_index = self._get_or_assign_tool_output_index(call_id)
 
             if call_id not in self._tool_args_by_call_id:
                 self._tool_args_by_call_id[call_id] = ""
                 self._sequence_number += 1
-                names = self._custom_tool_names
-                item_kwargs = build_tool_call_item_kwargs(call_id, tool_name, "", "in_progress", names)
+                item_kwargs = self._tool_call_item_kwargs(call_id, fn_name, "", "in_progress")
                 self._tool_item_id_by_call_id[call_id] = item_kwargs["id"]
-                if tool_namespace:
-                    item_kwargs["namespace"] = tool_namespace
                 event = OutputItemAddedEvent(
                     type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
                     output_index=output_index,
@@ -315,7 +320,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             else:
                 fn_name = str(getattr(fn, "name", "") or "")
                 fn_args = serialize_tool_call_arguments(getattr(fn, "arguments", ""))
-            tool_name, tool_namespace = self._responses_namespace_tool_call_fields(fn_name)
             web_search_call = self._web_search_calls.get(call_id)
             if web_search_call is not None:
                 if call_id not in self._queued_web_search_call_ids:
@@ -330,11 +334,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             if is_new_tool_call:
                 self._tool_args_by_call_id[call_id] = ""
                 self._sequence_number += 1
-                names = self._custom_tool_names
-                item_kwargs = build_tool_call_item_kwargs(call_id, tool_name, "", "in_progress", names)
+                item_kwargs = self._tool_call_item_kwargs(call_id, fn_name, "", "in_progress")
                 self._tool_item_id_by_call_id[call_id] = item_kwargs["id"]
-                if tool_namespace:
-                    item_kwargs["namespace"] = tool_namespace
                 event = OutputItemAddedEvent(
                     type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
                     output_index=output_index,
@@ -376,11 +377,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
             self._pending_tool_events.append(done_event)
 
             self._sequence_number += 1
-            names = self._custom_tool_names
-            item_kwargs = build_tool_call_item_kwargs(call_id, tool_name, final_args, "completed", names)
+            item_kwargs = self._tool_call_item_kwargs(call_id, fn_name, final_args, "completed")
             item_kwargs["id"] = self._tool_item_id_by_call_id.setdefault(call_id, item_kwargs["id"])
-            if tool_namespace:
-                item_kwargs["namespace"] = tool_namespace
             item_done_event = OutputItemDoneEvent(
                 type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
                 output_index=output_index,

litellm/responses/litellm_completion_transformation/transformation.py

@@ -110,6 +110,7 @@ NamespaceTool: TypeAlias = Mapping[str, object]
 ResponseTools: TypeAlias = Sequence[Mapping[str, object]] | None
 ChatToolParam: TypeAlias = ChatCompletionToolParam | OpenAIMcpServerTool
 NAMESPACE_DESCRIPTION_SEPARATOR: Final = "\n\n"
+NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS: Final = frozenset({"function", "custom"})
 
 
 @dataclass(frozen=True, slots=True)
@@ -1891,9 +1892,21 @@ class LiteLLMCompletionResponsesConfig:
         namespace_tool: NamespaceTool,
         nested: bool,
     ) -> ChatCompletionToolParam | None:
-        if nested and namespace_tool.get("type") != "function":
+        tool_type: Final = namespace_tool.get("type")
+        if nested and tool_type not in NAMESPACE_MEMBER_TYPES_WITH_CHAT_TOOLS:
             return None
 
+        raw_description: Final = str(namespace_tool.get("description") or "")
+        description: Final = (
+            f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}{raw_description}"
+            if nested and namespace_description and raw_description
+            else namespace_description
+            if nested and namespace_description
+            else raw_description
+        )
+        if nested and tool_type == "custom":
+            return convert_custom_tool_to_function_tool({**namespace_tool, "description": description})
+
         raw_parameters: Final = namespace_tool.get("parameters")
         parameters: Final = (
             MappingProxyType(raw_parameters) if isinstance(raw_parameters, Mapping) else MappingProxyType({})
@@ -1902,14 +1915,6 @@ class LiteLLMCompletionResponsesConfig:
             parameters if parameters and "type" in parameters else MappingProxyType({**parameters, "type": "object"})
         )
         tool_name: Final = str(namespace_tool.get("name") or "")
-        raw_description: Final = str(namespace_tool.get("description") or "")
-        description: Final = (
-            f"{namespace_description}{NAMESPACE_DESCRIPTION_SEPARATOR}{raw_description}"
-            if nested and namespace_description and raw_description
-            else namespace_description
-            if nested and namespace_description
-            else raw_description
-        )
         chat_tool_name: Final = f"{namespace}__{tool_name}" if nested else tool_name
         function: Final = ChatCompletionToolParamFunctionChunk(
             name=chat_tool_name,
@@ -2826,6 +2831,22 @@ class LiteLLMCompletionResponsesConfig:
                 if cache_write_tokens is not None
                 else MappingProxyType({})
             )
+            # The cost path reads the grounding counters off the input details, and a realtime
+            # session's usage is rebuilt from its own response.done, so dropping them here bills
+            # no per-query grounding fee at all.
+            grounding_request_counts: Final[Mapping[str, int]] = MappingProxyType(
+                {
+                    counter: count
+                    for counter, count in (
+                        ("web_search_requests", getattr(prompt_details, "web_search_requests", None)),
+                        (
+                            "google_maps_grounding_requests",
+                            getattr(prompt_details, "google_maps_grounding_requests", None),
+                        ),
+                    )
+                    if count is not None
+                }
+            )
             response_usage.input_tokens_details = InputTokensDetails(
                 cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
                 text_tokens=prompt_details.text_tokens,
@@ -2834,6 +2855,7 @@ class LiteLLMCompletionResponsesConfig:
                     cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
                 ),
                 **cache_write_extra,
+                **grounding_request_counts,
             )
 
         # Translate completion_tokens_details to output_tokens_details

litellm/responses/main.py

@@ -1,9 +1,10 @@
 import asyncio
 import contextvars
-from collections.abc import Coroutine, Generator, Iterable, Mapping
+from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence
 from contextlib import contextmanager
 from dataclasses import dataclass
 from functools import partial
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
 
 import httpx
@@ -15,7 +16,7 @@ from litellm._logging import verbose_logger
 from litellm.completion_extras.litellm_responses_transformation.transformation import (
     LiteLLMResponsesTransformationHandler,
 )
-from litellm.constants import request_timeout
+from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, request_timeout
 from litellm.integrations.anthropic_cache_control_hook import CARRY_UNMATCHED_MESSAGE_POINTS
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params
@@ -52,6 +53,7 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.responses.main import *
 from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import all_litellm_params
 from litellm.utils import (
     ProviderConfigManager,
     client,
@@ -408,6 +410,25 @@ def _bridges_to_chat_completions(
     return responses_api_provider_config is None or use_chat_completions_api is True
 
 
+def _bridge_kwargs(
+    kwargs: Mapping[str, object],
+    responses_api_provider_config: BaseResponsesAPIConfig | None,
+    allowed_openai_params: Sequence[str] | None,
+) -> Mapping[str, object]:
+    if responses_api_provider_config is None:
+        return kwargs
+    forwarded_keys: Final = frozenset(
+        (
+            *litellm.OPENAI_CHAT_COMPLETION_PARAMS,
+            *DEFAULT_CHAT_COMPLETION_PARAM_VALUES,
+            *all_litellm_params,
+            *GenericLiteLLMParams.model_fields,
+            *(allowed_openai_params or ()),
+        )
+    )
+    return MappingProxyType({key: value for key, value in kwargs.items() if key in forwarded_keys})
+
+
 _ResponsesCompatibilityFailure: TypeAlias = Literal["encrypted_task_unsupported"]
 
 
@@ -1281,6 +1302,7 @@ def responses(
             return _file_search_dispatch
 
         if _bridges_to_chat_completions(responses_api_provider_config, use_chat_completions_api):
+            bridge_kwargs: Final = _bridge_kwargs(kwargs, responses_api_provider_config, allowed_openai_params)
             return litellm_completion_transformation_handler.response_api_handler(
                 model=model,
                 input=input,
@@ -1292,7 +1314,7 @@ def responses(
                 extra_body=extra_body,
                 timeout=timeout if timeout is not None else request_timeout,
                 allowed_openai_params=allowed_openai_params,
-                **kwargs,
+                **bridge_kwargs,
             )
 
         # Get optional parameters for the responses API

litellm/responses/streaming_iterator.py

@@ -221,6 +221,13 @@ def _status_code_for_error_fields(error_type: str | None, error_code: str | None
     )
 
 
+def _mid_stream_fallback_eligible(mapped_exception: Exception) -> bool:
+    if isinstance(mapped_exception, litellm.ContentPolicyViolationError):
+        return True
+    status_code: Final = getattr(mapped_exception, "status_code", None)
+    return not isinstance(status_code, int) or status_code >= 500 or status_code == 429
+
+
 class BaseResponsesAPIStreamingIterator:
     """
     Base class for streaming iterators that process responses from the Responses API.
@@ -521,15 +528,8 @@ class BaseResponsesAPIStreamingIterator:
             getattr(self.completed_response, "response", None) if self.completed_response else None
         )
         error_info: Final = getattr(response_obj, "error", None) if response_obj else None
-        error_message, error_type, error_code = _error_event_fields(error_info)
         self._record_failed_response_usage(response_obj)
-        exception: Final = litellm.APIError(
-            status_code=_status_code_for_error_fields(error_type, error_code),
-            message=error_message,
-            llm_provider=self.custom_llm_provider or "",
-            model=self.model or "",
-        )
-        self._handle_failure(exception)
+        self._handle_failure(self._map_error_event_exception(error_info))
 
     def _record_failed_response_usage(self, response_obj: ResponsesAPIResponse | None) -> None:
         if response_obj is None or self.logging_obj is None:
@@ -551,6 +551,28 @@ class BaseResponsesAPIStreamingIterator:
             self.logging_obj._response_cost_calculator(result=response_obj) or 0.0
         )
 
+    def _map_error_event_exception(self, error_obj: object) -> Exception:
+        from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+        error_message, error_type, error_code = _error_event_fields(error_obj)
+        status_code: Final = _status_code_for_error_fields(error_type, error_code)
+        error_body: Final = {"message": error_message, "type": error_type, "code": error_code}
+        provider_exception: Final = BaseLLMException(
+            status_code=status_code,
+            message=f"Error code: {status_code} - {{'error': {error_body}}}",
+            body=error_body,
+        )
+        try:
+            return litellm.exception_type(
+                model=self.model or "",
+                custom_llm_provider=self.custom_llm_provider or "",
+                original_exception=provider_exception,
+                completion_kwargs={},
+                extra_kwargs={},
+            )
+        except Exception as mapped_exception:
+            return mapped_exception
+
     def _maybe_raise_for_error_event(self, result: object) -> None:
         chunk_type: Final = getattr(result, "type", None)
         if chunk_type not in ("error", "response.failed"):
@@ -562,15 +584,8 @@ class BaseResponsesAPIStreamingIterator:
             else getattr(result, "error", None)
         )
 
-        error_message, error_type, error_code = _error_event_fields(error_obj)
-        status_code: Final = _status_code_for_error_fields(error_type, error_code)
-        mapped_exception: Final = litellm.APIError(
-            status_code=status_code,
-            message=error_message,
-            llm_provider=self.custom_llm_provider or "",
-            model=self.model or "",
-        )
-        if 400 <= status_code < 500 and status_code != 429:
+        mapped_exception: Final = self._map_error_event_exception(error_obj)
+        if not _mid_stream_fallback_eligible(mapped_exception):
             raise mapped_exception
         raise MidStreamFallbackError(
             message=str(mapped_exception),

litellm/responses/utils.py

@@ -1183,6 +1183,10 @@ class ResponseAPILoggingUtils:
                         response_api_usage.input_tokens_details, "cached_tokens_details", None
                     ),
                     cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
+                    web_search_requests=getattr(response_api_usage.input_tokens_details, "web_search_requests", None),
+                    google_maps_grounding_requests=getattr(
+                        response_api_usage.input_tokens_details, "google_maps_grounding_requests", None
+                    ),
                 )
         completion_tokens_details: CompletionTokensDetailsWrapper | None = None
         output_tokens_details: Final[OutputTokensDetails | None] = getattr(

litellm/router.py

@@ -1622,6 +1622,24 @@ class Router:
             return
         await selector.async_pre_call_check(deployment, parent_otel_span)
 
+    def _bind_override_selector_to_request(
+        self, strategy: str, selector: RouterStrategySelector | None, request_kwargs: Mapping[str, object] | None
+    ) -> None:
+        if selector is None or request_kwargs is None or strategy in self._globally_registered_strategies():
+            return
+        logging_obj: Final = request_kwargs.get("litellm_logging_obj")
+        if isinstance(logging_obj, LiteLLMLogging):
+            logging_obj.add_dynamic_callback(selector)
+
+    def _globally_registered_strategies(self) -> frozenset[str]:
+        configured: Final = (
+            self.routing_strategy,
+            *(group.routing_strategy for group in self._routing_groups.values()),
+        )
+        return frozenset(
+            normalized for normalized in map(self._normalize_strategy, configured) if normalized is not None
+        )
+
     def _get_routing_context(
         self, model: str, request_kwargs: dict | None = None
     ) -> tuple[str | None, RouterStrategySelector | None]:
@@ -1647,7 +1665,9 @@ class Router:
         override: Final = self._get_request_routing_strategy_override(request_kwargs)
         if override is not None:
             verbose_router_logger.debug("routing_group=request-override model=%s strategy=%s", model, override)
-            return override, self._get_override_strategy_selector(override)
+            override_selector: Final = self._get_override_strategy_selector(override)
+            self._bind_override_selector_to_request(override, override_selector, request_kwargs)
+            return override, override_selector
 
         group_name: Final = model if self.get_routing_group(model) is not None else self._model_to_group.get(model)
         if group_name is None:
@@ -3268,8 +3288,15 @@ class Router:
                         kwargs=initial_kwargs,
                         metadata_variable_name="litellm_metadata",
                     )
+                    # The content-policy dispatch branch matches on the trigger's own type, so a refusal's
+                    # MidStreamFallbackError envelope is unwrapped here or the wrong fallback list is consulted.
+                    fallback_trigger: Final[Exception] = (
+                        e.original_exception
+                        if isinstance(e.original_exception, litellm.ContentPolicyViolationError)
+                        else e
+                    )
                     fallback_response = await self.async_function_with_fallbacks_common_utils(
-                        e=e,
+                        e=fallback_trigger,
                         disable_fallbacks=False,
                         fallbacks=fallbacks,
                         context_window_fallbacks=context_window_fallbacks,
@@ -4842,6 +4869,7 @@ class Router:
                 model=model,
                 messages=messages,
                 specific_deployment=kwargs.pop("specific_deployment", None),
+                request_kwargs=kwargs,
             )
 
             data: Final = deployment["litellm_params"].copy()
@@ -5156,13 +5184,11 @@ class Router:
             return healthy_deployments[0]
 
         # Use simple_shuffle for weighted selection
-        return cast(
-            GuardrailTypedDict,
-            simple_shuffle(
-                llm_router_instance=self,
-                healthy_deployments=healthy_deployments,
-                model=guardrail_name,
-            ),
+        return simple_shuffle(
+            resolve_model_alias=self._get_model_from_alias,
+            healthy_deployments=healthy_deployments,
+            model=guardrail_name,
+            request_kwargs=None,
         )
 
     async def _ageneric_api_call_with_fallbacks(self, model: str, original_function: Callable, **kwargs):
@@ -8371,7 +8397,8 @@ class Router:
 
     def log_retry(self, kwargs: dict, e: Exception) -> dict:
         """
-        When a retry or fallback happens, record which model group, deployment and attempt just failed and why
+        When a retry or fallback happens, record which model group, deployment and attempt just failed and why,
+        and count it toward the request-wide num_retries_per_request cap
         """
         from litellm.types.router import RetryAttemptRecord
 
@@ -8395,7 +8422,10 @@ class Router:
             else ()
         )
         breadcrumbs: Final = (*kept_breadcrumbs, attempt_record)
+        earlier: Final = request_metadata.get("request_retry_count")
+        request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1
         kwargs[_metadata_var]["previous_models"] = breadcrumbs  # rebind-ok: the logging object already holds this dict
+        kwargs[_metadata_var]["request_retry_count"] = request_retry_count  # rebind-ok: same dict, read by the cap
         return kwargs
 
     def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int:
@@ -13038,9 +13068,10 @@ class Router:
             start_time: Final = time.time()
             if strategy == "simple-shuffle":
                 return simple_shuffle(
-                    llm_router_instance=self,
+                    resolve_model_alias=self._get_model_from_alias,
                     healthy_deployments=healthy_deployments,
                     model=model,
+                    request_kwargs=request_kwargs,
                 )
             deployment: Final = await self._select_deployment_async(
                 strategy=strategy,
@@ -13183,9 +13214,10 @@ class Router:
             start_time: Final = time.perf_counter()
             if strategy == "simple-shuffle":
                 return simple_shuffle(
-                    llm_router_instance=self,
+                    resolve_model_alias=self._get_model_from_alias,
                     healthy_deployments=pass_through_deployments,
                     model=model,
+                    request_kwargs=request_kwargs,
                 )
             deployment: Final = await self._select_deployment_async(
                 strategy=strategy,
@@ -13463,7 +13495,7 @@ class Router:
     async def async_pre_routing_hook(
         self,
         model: str,
-        request_kwargs: dict,
+        request_kwargs: dict[str, object],
         messages: list[dict[str, Any]] | None = None,
         input: str | list | None = None,
         specific_deployment: bool | None = False,
@@ -13511,6 +13543,18 @@ class Router:
             )
             return None
 
+        from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
+
+        await authorize_member_auto_router_inference(
+            deployment=self._selected_strategy_marker_deployment(
+                model=registered_model_name,
+                strategy_tags=selected_strategy.tags,
+                request_kwargs=request_kwargs,
+            ),
+            request_kwargs=request_kwargs,
+            llm_router=self,
+        )
+
         from litellm.proxy.guardrails.auto_router_compression import (
             messages_for_routing,
             model_hop_compression_armed,
@@ -13610,25 +13654,34 @@ class Router:
 
         return pre_routing_hook_response
 
+    def _selected_strategy_marker_deployment(
+        self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object]
+    ) -> DeploymentTypedDict | None:
+        markers: Final = tuple(
+            deployment
+            for deployment in self.deployments_for_request(model, request_kwargs)
+            if "model" in deployment["litellm_params"]
+            and str(deployment["litellm_params"]["model"]).startswith(AUTO_ROUTER_MODEL_PREFIX)
+        )
+        tag_matched: Final = tuple(
+            deployment
+            for deployment in markers
+            if (tuple(deployment["litellm_params"]["tags"] or ()) if "tags" in deployment["litellm_params"] else ())
+            == strategy_tags
+        )
+        return tag_matched[0] if tag_matched else (markers[0] if markers else None)
+
     def _forwardable_alias_marker_params(
         self, model: str, strategy_tags: tuple[str, ...], request_kwargs: Mapping[str, object]
     ) -> tuple[tuple[str, object], ...]:
-        marker_params: Final = tuple(
-            litellm_params
-            for deployment in self.deployments_for_request(model, request_kwargs)
-            if str((litellm_params := deployment["litellm_params"]).get("model", "")).startswith(
-                AUTO_ROUTER_MODEL_PREFIX
-            )
+        marker: Final = self._selected_strategy_marker_deployment(
+            model=model, strategy_tags=strategy_tags, request_kwargs=request_kwargs
         )
-        tag_matched: Final = tuple(
-            params for params in marker_params if tuple(params.get("tags") or ()) == strategy_tags
-        )
-        selected: Final = tag_matched[0] if tag_matched else (marker_params[0] if marker_params else None)
-        if selected is None:
+        if marker is None:
             return ()
         return tuple(
             (key, value)
-            for key, value in selected.items()
+            for key, value in marker["litellm_params"].items()
             if key not in _ALIAS_PARAMS_NEVER_FORWARDED
             and key not in CustomPricingLiteLLMParams.model_fields
             and value is not None
@@ -13881,9 +13934,10 @@ class Router:
             # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
             ############## Check 'weight' param set for weighted pick #################
             return simple_shuffle(
-                llm_router_instance=self,
+                resolve_model_alias=self._get_model_from_alias,
                 healthy_deployments=healthy_deployments,
                 model=model,
+                request_kwargs=request_kwargs,
             )
         deployment: Final = self._select_deployment_sync(
             strategy=strategy,
@@ -13951,6 +14005,7 @@ class Router:
             messages=messages,
             input=input,
             specific_deployment=specific_deployment,
+            request_kwargs=request_kwargs,
         )
 
         strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
@@ -14033,9 +14088,10 @@ class Router:
         # 6. Apply load balancing strategy
         if strategy == "simple-shuffle":
             return simple_shuffle(
-                llm_router_instance=self,
+                resolve_model_alias=self._get_model_from_alias,
                 healthy_deployments=pass_through_deployments,
                 model=model,
+                request_kwargs=request_kwargs,
             )
         deployment: Final = self._select_deployment_sync(
             strategy=strategy,

litellm/router_strategy/complexity_router/complexity_router.py

@@ -16,6 +16,8 @@ Inspired by ClawRouter: https://github.com/BlockRunAI/ClawRouter
 from __future__ import annotations
 
 import asyncio
+import hashlib
+import json
 import random
 import re
 import time
@@ -28,6 +30,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
 from pydantic import BaseModel, TypeAdapter, ValidationError, create_model
 
 from litellm._logging import verbose_router_logger
+from litellm.caching.affinity_cache import claim_affinity_pin
 from litellm.constants import (
     EMPTY_MAPPING,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
@@ -55,6 +58,7 @@ from litellm.router_strategy.complexity_router.tier_predictor import (
     TierSuccessPredictor,
     resolve_tier_artifact,
 )
+from litellm.router_utils.pre_call_checks.deployment_affinity_check import DeploymentAffinityCheck
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionImageObject,
@@ -1119,10 +1123,10 @@ class _ContextWindowPlacement(NamedTuple):
 
 class _SessionAffinityPin(NamedTuple):
     model: str
-    tier: ComplexityTier | None
+    tier: ComplexityTier | str | None
 
 
-def _parse_session_affinity_pin(value: object) -> _SessionAffinityPin | None:
+def _parse_session_affinity_pin(value: object, active_tiers: tuple[str, ...]) -> _SessionAffinityPin | None:
     if isinstance(value, str):
         return _SessionAffinityPin(model=value, tier=None)
     parts: Final[tuple[object, object] | None] = (
@@ -1137,8 +1141,11 @@ def _parse_session_affinity_pin(value: object) -> _SessionAffinityPin | None:
     model, tier_value = parts
     if not isinstance(model, str):
         return None
-    tier: Final = ComplexityTier(tier_value) if isinstance(tier_value, str) else None
-    return _SessionAffinityPin(model=model, tier=tier)
+    if tier_value is None:
+        return _SessionAffinityPin(model=model, tier=None)
+    if not isinstance(tier_value, str) or tier_value not in active_tiers:
+        return None
+    return _SessionAffinityPin(model=model, tier=_built_in_tier_or_none(tier_value) or tier_value)
 
 
 def _session_affinity_cache_value(model: str, tier: ComplexityTier | str | None) -> Mapping[str, str | None]:
@@ -1195,6 +1202,10 @@ class ComplexityRouter(CustomLogger):
         if default_model:
             self.config.default_model = default_model
 
+        self._tier_affinity_config = hashlib.sha256(
+            self.config.model_dump_json(include=MappingProxyType({"tiers": True, "tier_model_configs": True})).encode()
+        ).hexdigest()
+
         # Checked here rather than on the config model because the deployment's
         # complexity_router_default_model arrives outside complexity_router_config and is
         # applied just above, so a validator on the model would reject a deployment that
@@ -2259,6 +2270,51 @@ class ComplexityRouter(CustomLogger):
     def _tier_pools(self) -> dict[str, list[str]]:
         return {tier: (models if isinstance(models, list) else [models]) for tier, models in self.config.tiers.items()}
 
+    async def _pin_model_for_tier(
+        self,
+        tier: ComplexityTier | str,
+        model: str,
+        candidates: tuple[str, ...],
+        request_kwargs: dict[str, object],  # mutable-ok: adaptive feedback metadata must follow the selected model
+        retained_pin: _SessionAffinityPin | None = None,
+    ) -> str:
+        if not self._uses_deployment_pin or model not in candidates:
+            return model
+        retained_model: Final = (
+            retained_pin.model
+            if retained_pin is not None
+            and retained_pin.tier is not None
+            and _tier_name(retained_pin.tier) == _tier_name(tier)
+            else None
+        )
+        if retained_model is not None and retained_model in candidates:
+            self._restamp_adaptive_choice(request_kwargs, model, retained_model)
+            return retained_model
+        session_id: Final = self._get_session_id_from_request_kwargs(request_kwargs)
+        if session_id is None:
+            return model
+        caller: Final = DeploymentAffinityCheck.get_user_key_from_request_kwargs(request_kwargs)
+        identity: Final = (self.model_name, self._tier_affinity_config, caller, session_id, _tier_name(tier))
+        cache_identity: Final = (
+            (*identity, ("replay_fallback", retained_model)) if retained_model is not None else identity
+        )
+        cache_key: Final = (
+            "complexity_router_tier_model_affinity:v1:"
+            + hashlib.sha256(json.dumps(cache_identity).encode()).hexdigest()
+        )
+        winner: Final = await claim_affinity_pin(
+            self.litellm_router_instance.cache,
+            cache_key,
+            MappingProxyType({"model": model}),
+            self.config.session_affinity_ttl_seconds,
+            eligible_values=tuple(MappingProxyType({"model": candidate}) for candidate in candidates),
+        )
+        pinned: Final[object] = winner.get("model") if isinstance(winner, Mapping) else None
+        if not isinstance(pinned, str) or pinned not in candidates:
+            return model
+        self._restamp_adaptive_choice(request_kwargs, model, pinned)
+        return pinned
+
     async def _pick_model_for_tier(
         self,
         tier: ComplexityTier | str,
@@ -2266,11 +2322,18 @@ class ComplexityRouter(CustomLogger):
         resolved_messages: list[dict[str, Any]] | None,
         request_kwargs: dict,
         allowed_models: tuple[str, ...] | None = None,
+        retained_pin: _SessionAffinityPin | None = None,
     ) -> str:
         if not self.config.plugins:
-            if allowed_models is not None:
-                return self._pick_from_tier_value(allowed_models, _tier_name(tier))
-            return self.get_model_for_tier(tier)
+            candidates: Final = (
+                allowed_models if allowed_models is not None else tuple(self._tier_pools().get(_tier_name(tier), ()))
+            )
+            selected: Final = (
+                self._pick_from_tier_value(allowed_models, _tier_name(tier))
+                if allowed_models is not None
+                else self.get_model_for_tier(tier)
+            )
+            return await self._pin_model_for_tier(tier, selected, candidates, request_kwargs, retained_pin)
 
         from litellm.types.router import RoutingContext
 
@@ -2369,6 +2432,40 @@ class ComplexityRouter(CustomLogger):
         self._adaptive_chosen_model_key = ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY
         return self.adaptive_router
 
+    def _adaptive_candidate_models(
+        self,
+        classified_tier: ComplexityTier | str,
+        hard_floor: ComplexityTier | str | None = None,
+        hard_ceiling: ComplexityTier | str | None = None,
+        fit_filter: frozenset[str] | None = None,
+    ) -> tuple[str, ...]:
+        pools: Final = self._tier_pools()
+        candidates: Final = (
+            tuple(pools.get(_tier_name(classified_tier), ()))
+            if self.config.adaptive_eligible == "classified_tier"
+            else tuple(dict.fromkeys(chain.from_iterable(pools.values())))
+        )
+        floor: Final = self._active_tier_severity(hard_floor) if hard_floor is not None else None
+        ceiling: Final = self._active_tier_severity(hard_ceiling) if hard_ceiling is not None else None
+        return tuple(
+            model
+            for model in _allowed(candidates, fit_filter)
+            if (
+                floor is None
+                or any(
+                    self._active_tier_severity(tier) >= floor
+                    for tier in self._model_tiers.get(model, (classified_tier,))
+                )
+            )
+            and (
+                ceiling is None
+                or any(
+                    self._active_tier_severity(tier) <= ceiling
+                    for tier in self._model_tiers.get(model, (classified_tier,))
+                )
+            )
+        )
+
     def _soft_floor_pick(
         self,
         classified_tier: ComplexityTier | str,
@@ -2436,34 +2533,17 @@ class ComplexityRouter(CustomLogger):
                         ],
                     }
             return chosen_model
-        if self.config.adaptive_eligible == "classified_tier":
-            candidates = list(classified_candidates)
-            if not candidates:
-                return self._fitting_tier_fallback(classified_tier, fit_filter)
-        else:
-            candidates = list(_allowed(tuple(adaptive.config.available_models), fit_filter))
+        candidates: Final = self._adaptive_candidate_models(classified_tier, fit_filter=fit_filter)
 
         all_costs: Final = [adaptive.model_to_cost.get(m, 0.0) for m in candidates]
         quality_weight: Final = self.config.adaptive_weights.quality
         cost_weight: Final = self.config.adaptive_weights.cost
         penalty_weight: Final = self.config.tier_distance_penalty
 
-        floor_severity: Final = self._active_tier_severity(hard_floor) if hard_floor is not None else None
-        ceiling_severity: Final = self._active_tier_severity(hard_ceiling) if hard_ceiling is not None else None
         best_model: str | None = None
         best_score = float("-inf")
         candidate_scores: Final[list[dict[str, object]]] = []
-        for model in candidates:
-            if floor_severity is not None and all(
-                self._active_tier_severity(model_tier) < floor_severity
-                for model_tier in self._model_tiers.get(model, (classified_tier,))
-            ):
-                continue
-            if ceiling_severity is not None and all(
-                self._active_tier_severity(model_tier) > ceiling_severity
-                for model_tier in self._model_tiers.get(model, (classified_tier,))
-            ):
-                continue
+        for model in self._adaptive_candidate_models(classified_tier, hard_floor, hard_ceiling, fit_filter):
             cell = adaptive._cells[(request_type, model)]
             quality_sample = thompson_sample(cell)
             cost_score = normalized_cost(adaptive.model_to_cost.get(model, 0.0), all_costs)
@@ -2644,8 +2724,6 @@ class ComplexityRouter(CustomLogger):
         """Prompt content the resolved message list never carries: the Responses API's
         `instructions`, the /v1/messages top-level `system` block, and tool definitions.
         A coding agent's context is dominated by these."""
-        import json
-
         instructions: Final = request_kwargs.get("instructions")
         proxy_request: Final = request_kwargs.get("proxy_server_request")
         body: Final = proxy_request.get("body") if isinstance(proxy_request, Mapping) else None
@@ -2831,19 +2909,21 @@ class ComplexityRouter(CustomLogger):
         )
         return higher_tiers[0] if higher_tiers else tier
 
-    def _escalated_pin(self, pinned_model: str) -> str | None:
+    def _escalated_pin(self, pinned_model: str, tier: ComplexityTier | str | None = None) -> _SessionAffinityPin | None:
         """Bump a session's pinned model to the next-higher configured tier.
 
         Returns None when the pin no longer maps to any configured tier, signalling
         a full reclassification instead.
         """
-        pinned_tier: Final = self._tier_for_model(pinned_model)
+        pinned_tier: Final = tier if tier is not None else self._tier_for_model(pinned_model)
         if pinned_tier is None:
             return None
         escalated_tier: Final = self._escalate_tier(pinned_tier)
         if escalated_tier == pinned_tier:
-            return pinned_model
-        return self.get_model_for_tier(escalated_tier)
+            return _SessionAffinityPin(pinned_model, pinned_tier)
+        return _SessionAffinityPin(
+            self.get_model_for_tier(escalated_tier), _built_in_tier_or_none(_tier_name(escalated_tier))
+        )
 
     def _vision_verdicts(self, model_name: str) -> tuple[bool | None, ...]:
         """Declared vision support per deployment serving the name: True, False, or None when
@@ -2907,6 +2987,7 @@ class ComplexityRouter(CustomLogger):
         resolved_messages: Sequence[Mapping[str, object]] | None,
         request_kwargs: dict,  # mutable-ok: same shape the hook receives
         context_fit: _RequestContextFit | None = None,
+        retained_pin: _SessionAffinityPin | None = None,
     ) -> PreRoutingHookResponse:
         """Replace a routed model that cannot accept this request's image input.
 
@@ -2955,6 +3036,7 @@ class ComplexityRouter(CustomLogger):
                 repick_messages,  # pyright: ignore[reportArgumentType]  # hook-resolved message dicts; the pick only reads them
                 request_kwargs,
                 allowed_models=tuple(entry for entry in pools.get(capable, ()) if entry in eligible),
+                retained_pin=retained_pin,
             )
         elif self._modality_default_model_usable(request_kwargs, resolved_messages, eligible):
             new_tier = None
@@ -3098,6 +3180,7 @@ class ComplexityRouter(CustomLogger):
         resolved_messages: Sequence[Mapping[str, object]] | None,
         request_kwargs: dict,  # mutable-ok: same shape the hook receives
         context_fit: _RequestContextFit | None = None,
+        retained_pin: _SessionAffinityPin | None = None,
     ) -> PreRoutingHookResponse:
         """Try compatible tier recovery before the default, preserving request policy and fit."""
         decision: Final = response.routing_decision
@@ -3155,6 +3238,7 @@ class ComplexityRouter(CustomLogger):
                         repick_messages,  # pyright: ignore[reportArgumentType]  # hook-resolved message dicts; the pick only reads them
                         request_kwargs,
                         allowed_models=live,
+                        retained_pin=retained_pin,
                     )
                 except ValueError as exc:
                     verbose_router_logger.debug(
@@ -3247,8 +3331,13 @@ class ComplexityRouter(CustomLogger):
         """The adaptive feedback loop reads its chosen-model marker from request metadata; a
         gate rewrite must move the marker with the model or rewards land on the displaced one."""
         metadata: Final = request_kwargs.get("metadata")
-        if isinstance(metadata, dict) and metadata.get("adaptive_router_chosen_model") == old_model:
+        if not isinstance(metadata, dict):
+            return
+        if metadata.get("adaptive_router_chosen_model") == old_model:
             metadata["adaptive_router_chosen_model"] = new_model
+        decision: Final = metadata.get("adaptive_router_decision")
+        if isinstance(decision, dict) and decision.get("chosen_model") == old_model:
+            decision["chosen_model"] = new_model
 
     def _lexical_tier_override(self, user_message: str) -> KeywordOverride | None:
         """When keyword_tier_rules match literally, the most-severe matched tier wins.
@@ -3561,25 +3650,42 @@ class ComplexityRouter(CustomLogger):
 
         if cache_key is not None and pin_replay_allowed:
             pinned_value: Final = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
-            pinned_pin: Final = _parse_session_affinity_pin(pinned_value)
+            pinned_pin: Final = _parse_session_affinity_pin(pinned_value, self.config.tier_names())
             if pinned_pin is not None:
-                routed_model: str | None = pinned_pin.model
-                pin_escalation_keyword: str | None = None
-                if self.escalation_keywords:
-                    user_message: Final = (
-                        _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None
+                user_message: Final = _newest_turn_ask(resolved_messages, marker_pairs) if resolved_messages else None
+                pin_escalation_keyword: Final = (
+                    self._matched_escalation_keyword(user_message) if user_message is not None else None
+                )
+                selected_pin: Final = (
+                    self._escalated_pin(pinned_pin.model, pinned_pin.tier)
+                    if pin_escalation_keyword is not None
+                    else _SessionAffinityPin(
+                        pinned_pin.model,
+                        pinned_pin.tier if pinned_pin.tier is not None else self._tier_for_model(pinned_pin.model),
                     )
-                    if user_message is not None:
-                        pin_escalation_keyword = self._matched_escalation_keyword(user_message)
-                    if pin_escalation_keyword is not None:
-                        routed_model = self._escalated_pin(pinned_pin.model)
-                if routed_model is not None:
-                    escalated: Final = routed_model != pinned_pin.model
-                    resolved_pin_tier: Final = (
-                        pinned_pin.tier
-                        if not escalated and pinned_pin.tier is not None
-                        else self._tier_for_model(routed_model)
+                )
+                if selected_pin is not None:
+                    escalated: Final = selected_pin.model != pinned_pin.model or (
+                        pin_escalation_keyword is not None
+                        and pinned_pin.tier is not None
+                        and selected_pin.tier != pinned_pin.tier
                     )
+                    resolved_pin_tier: Final = selected_pin.tier
+                    session_model: Final = (
+                        await self._pin_model_for_tier(
+                            resolved_pin_tier,
+                            selected_pin.model,
+                            tuple(self._tier_pools().get(_tier_name(resolved_pin_tier), ())),
+                            request_kwargs,
+                        )
+                        if escalated and resolved_pin_tier is not None
+                        else selected_pin.model
+                    )
+                    retained_pin: Final = _SessionAffinityPin(session_model, resolved_pin_tier)
+                    if resolved_pin_tier is not None:
+                        await self._pin_model_for_tier(
+                            resolved_pin_tier, session_model, (session_model,), request_kwargs
+                        )
                     # The floor outranks the pin because plan mode is a transient state of the
                     # session, not a request to move it: the turns carrying the sentinel route at
                     # the floor, and the stored pin deliberately keeps the session's own model so
@@ -3590,16 +3696,28 @@ class ComplexityRouter(CustomLogger):
                     plan_floored: Final = (
                         pinned_tier is not None and self._apply_plan_mode_floor(pinned_tier) != pinned_tier
                     )
-                    session_model: Final = routed_model
-                    if plan_floored and pinned_tier is not None:
-                        routed_model = self.get_model_for_tier(self._apply_plan_mode_floor(pinned_tier))
-                    pin_source_tier: Final = self._tier_for_model(routed_model)
+                    floor_model: Final = (
+                        await self._pick_model_for_tier(
+                            self._apply_plan_mode_floor(pinned_tier),
+                            messages,
+                            resolved_messages,
+                            request_kwargs,
+                            retained_pin=retained_pin,
+                        )
+                        if plan_floored and pinned_tier is not None
+                        else session_model
+                    )
+                    pin_source_tier: Final = (
+                        self._apply_plan_mode_floor(pinned_tier)
+                        if plan_floored and pinned_tier is not None
+                        else resolved_pin_tier
+                    )
                     pin_placement: Final = (
                         await self._context_window_placement(
                             pin_source_tier,
                             resolved_messages,
                             request_kwargs,
-                            pool_override=(routed_model,),
+                            pool_override=(floor_model,),
                             context_fit=context_fit,
                         )
                         if pin_source_tier is not None
@@ -3612,11 +3730,18 @@ class ComplexityRouter(CustomLogger):
                         and _tier_name(pin_placement.tier) != _tier_name(pin_source_tier)
                         else None
                     )
-                    if pin_placement is not None and pin_context_original_tier is not None:
-                        # The stored pin below keeps the session's own model on purpose.
-                        routed_model = self._pick_from_tier_value(
-                            pin_placement.allowed_models, _tier_name(pin_placement.tier)
+                    routed_model: Final = (
+                        await self._pick_model_for_tier(
+                            pin_placement.tier,
+                            messages,
+                            resolved_messages,
+                            request_kwargs,
+                            allowed_models=pin_placement.allowed_models,
+                            retained_pin=retained_pin,
                         )
+                        if pin_placement is not None and pin_context_original_tier is not None
+                        else floor_model
+                    )
                     # Refresh the TTL on every hit so an active session doesn't lose its
                     # pin mid-conversation just because it outlives the original write.
                     await self.litellm_router_instance.cache.async_set_cache(
@@ -3644,7 +3769,7 @@ class ComplexityRouter(CustomLogger):
                     routed_pin_tier: Final = (
                         pin_placement.tier
                         if pin_placement is not None and pin_context_original_tier is not None
-                        else (self._tier_for_model(routed_model) if plan_floored else resolved_pin_tier)
+                        else pin_source_tier
                     )
                     session_tier_litellm_params: Final = self._litellm_params_for_model(routed_pin_tier, routed_model)
                     has_original_messages: Final = messages is not None and len(messages) > 0
@@ -3671,12 +3796,14 @@ class ComplexityRouter(CustomLogger):
                                 resolved_messages,
                                 request_kwargs,
                                 context_fit,
+                                retained_pin,
                             ),
                             messages,
                             input,
                             resolved_messages,
                             request_kwargs,
                             context_fit,
+                            retained_pin,
                         )
                     )
 
@@ -3961,13 +4088,21 @@ class ComplexityRouter(CustomLogger):
             housekeeping_ceiling: Final = tier if outcome.cause == "housekeeping" else None
             # A context-escalated tier becomes the hard floor: a floor the bandit can slide
             # under is not a floor.
-            routed_model = self._soft_floor_pick(
+            adaptive_floor: Final = tier if context_original_tier is not None else plan_floor
+            adaptive_fit: Final = context_placement.holdable_models if context_placement is not None else None
+            sampled_model: Final = self._soft_floor_pick(
                 tier,
                 ask,
                 request_kwargs,
-                hard_floor=tier if context_original_tier is not None else plan_floor,
+                hard_floor=adaptive_floor,
                 hard_ceiling=housekeeping_ceiling,
-                fit_filter=context_placement.holdable_models if context_placement is not None else None,
+                fit_filter=adaptive_fit,
+            )
+            routed_model = await self._pin_model_for_tier(  # rebind-ok: reuse the eligible tier winner
+                tier,
+                sampled_model,
+                self._adaptive_candidate_models(tier, adaptive_floor, housekeeping_ceiling, adaptive_fit),
+                request_kwargs,
             )
             adaptive: Final = self._ensure_adaptive_router()
             if adaptive is not None:

litellm/router_strategy/complexity_router/config.py

@@ -1256,20 +1256,16 @@ class ComplexityRouterConfig(BaseModel):
     deployment_affinity: bool = Field(
         default=True,
         description=(
-            "When True and a session_id is resolvable on the request, pin the deployment chosen "
-            "inside each routed model group and reuse it whenever the session returns to that "
-            "group, without pinning which group the session routes to. Independent of "
-            "session_affinity, which pins the model group instead (and always carries this "
-            "deployment pin with it): with session_affinity off, "
-            "every turn is still classified on its own merits while a session that escalates to a "
-            "stronger tier and comes back still lands on the deployment it used before, which is "
-            "what keeps a provider prompt cache warm. Pins are held per model group, so switching "
-            "tiers does not disturb the pin left behind in the previous group. On by default "
-            "because re-shuffling a conversation across deployments of the same model discards "
-            "that cache for no benefit; set False to keep every turn load-balanced across the "
-            "group, which is what a deployment set with tight per-deployment rate limits wants. "
-            "Inert when no session_id is resolvable, since there is nothing to key a pin on, and "
-            "suppressed when plugins are configured, for the same reason session_affinity is."
+            "When True and a client session_id is resolvable, reuse the session's chosen model "
+            "for each classified tier and its deployment within each model group. With "
+            "session_affinity off, every turn is still classified: moving to another tier leaves "
+            "the previous tier's model pin intact for a later return. Pins yield to current "
+            "candidate, context, modality, and availability constraints. Adaptive selection chooses "
+            "the initial model from its eligible pool, then reuses that choice per tier. This "
+            "reduces avoidable provider prompt-cache misses; it does not guarantee cache hits. "
+            "Set False to select models and load-balance deployments on every turn, unless "
+            "session_affinity or user_turn classification requires a pin. Inert without a client "
+            "session_id and suppressed when plugins are configured."
         ),
     )
     session_affinity_ttl_seconds: int = Field(
@@ -1277,7 +1273,7 @@ class ComplexityRouterConfig(BaseModel):
         gt=0,
         description=(
             "TTL for the session affinity pin; refreshed on every cache hit. Bounds both the "
-            "session_affinity model pin and the deployment_affinity deployment pin, so it measures "
+            "session_affinity model pin and the deployment_affinity per-tier model and deployment pins, so it measures "
             "idle time for the session's routing decisions rather than total session length"
         ),
     )

litellm/router_strategy/simple_shuffle.py

@@ -1,71 +1,67 @@
-"""
-Returns a random deployment from the list of healthy deployments.
+"""Choose among eligible deployments using request weights, then global metrics."""
 
-If weights are provided, it will return a deployment based on the weights.
-
-"""
+from __future__ import annotations
 
+import logging
 import random
-from typing import TYPE_CHECKING, Any, Final
+from collections.abc import Callable, Mapping, Sequence
+from itertools import chain
+from typing import Final, TypeVar
 
-from litellm._logging import verbose_router_logger
+from litellm.types.router_weights import validate_router_weights
 
-if TYPE_CHECKING:
-    from litellm.router import Router as _Router
+_DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object])
+_ROUTER_LOGGER: Final = logging.getLogger("LiteLLM Router")
 
-    LitellmRouter = _Router
-else:
-    LitellmRouter = Any
+
+def _metric_weight(deployment: Mapping[str, object], metric: str) -> float:
+    params: Final = deployment.get("litellm_params")
+    value: Final = params.get(metric) if isinstance(params, Mapping) else None
+    if value is None:
+        return 0.0
+    if isinstance(value, (int, float)):
+        return float(value)
+    raise TypeError(f"Deployment {metric} must be numeric")
+
+
+def _scoped_weights(
+    deployments: Sequence[Mapping[str, object]],
+    model: str,
+    request_kwargs: Mapping[str, object] | None,
+) -> tuple[float, ...]:
+    settings: Final = validate_router_weights((request_kwargs or {}).get("_router_weights"))
+    model_weights: Final = settings.get(model) if settings is not None else None
+    if not model_weights:
+        return ()
+    return tuple(
+        model_weights.get(str(info.get("id")), 0.0) if isinstance(info, Mapping) else 0.0
+        for deployment in deployments
+        for info in (deployment.get("model_info"),)
+    )
 
 
 def simple_shuffle(
-    llm_router_instance: LitellmRouter,
-    healthy_deployments: list[Any] | dict[Any, Any],
+    resolve_model_alias: Callable[[str], str | None],
+    healthy_deployments: Sequence[_DeploymentT],
     model: str,
-) -> dict:
-    """
-    Returns a random deployment from the list of healthy deployments.
-
-    If weights are provided, it will return a deployment based on the weights.
-
-    If users pass `rpm` or `tpm`, we do a random weighted pick - based on `rpm`/`tpm`.
-
-    Args:
-        llm_router_instance: LitellmRouter instance
-        healthy_deployments: List of healthy deployments
-        model: Model name
-
-    Returns:
-        Dict: A single healthy deployment
-    """
-
-    ############## Check if 'weight' or 'rpm' or 'tpm' param set for a weighted pick #################
-    for weight_by in ["weight", "rpm", "tpm"]:
-        if any(m["litellm_params"].get(weight_by) is not None for m in healthy_deployments):
-            weights = [m["litellm_params"].get(weight_by, 0) for m in healthy_deployments]
-            verbose_router_logger.debug("\nweight %s", weights)
-            total_weight = sum(weights)
-            if total_weight <= 0:
-                # All remaining candidates have weight 0 for this metric (e.g.
-                # after a weighted-failover exclusion left only zero-weight
-                # backups). Skip to the next metric (rpm/tpm) which may still
-                # provide a meaningful weighted pick; if none do, we fall
-                # through to the uniform random pick at the end.
-                continue
-            weights = [weight / total_weight for weight in weights]
-            verbose_router_logger.debug("\n weights %s by %s", weights, weight_by)
-            # Perform weighted random pick
-            selected_index = random.choices(range(len(weights)), weights=weights)[0]
-            verbose_router_logger.debug("\n selected index, %s", selected_index)
-            deployment = healthy_deployments[selected_index]
-            verbose_router_logger.info(
-                "get_available_deployment for model: %s, Selected deployment: %s for model: %s",
-                model,
-                llm_router_instance.print_deployment(deployment) or deployment[0],
-                model,
-            )
-            return deployment or deployment[0]
-
-    ############## No RPM/TPM passed, we do a random pick #################
-    item: Final = random.choice(healthy_deployments)
-    return item or item[0]
+    request_kwargs: Mapping[str, object] | None,
+) -> _DeploymentT:
+    resolved_model: Final = resolve_model_alias(model) or model
+    weight_sets: Final = chain(
+        (_scoped_weights(healthy_deployments, resolved_model, request_kwargs),),
+        (
+            tuple(_metric_weight(deployment, metric) for deployment in healthy_deployments)
+            for metric in ("weight", "rpm", "tpm")
+        ),
+    )
+    for weights in weight_sets:
+        largest = max(weights, default=0.0)
+        if largest <= 0:
+            continue
+        normalized = tuple(weight / largest for weight in weights)
+        if sum(normalized) <= 0:
+            continue
+        selected = random.choices(healthy_deployments, weights=normalized)[0]
+        _ROUTER_LOGGER.info("Selected deployment for model %s: %s", model, selected.get("model_info"))
+        return selected
+    return random.choice(healthy_deployments)

litellm/router_utils/pre_call_checks/deployment_affinity_check.py

@@ -13,13 +13,13 @@ where routing to a consistent deployment is still beneficial.
 """
 
 import hashlib
-import json
 from collections.abc import Mapping, Sequence
 from typing import Any, Final, cast
 
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_router_logger
+from litellm.caching.affinity_cache import claim_affinity_pin, claim_affinity_pin_in_memory, set_local_affinity_pin
 from litellm.caching.dual_cache import DualCache
 from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
 from litellm.integrations.custom_logger import CustomLogger, Span
@@ -28,8 +28,8 @@ from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import CallTypes
 
 
-class DeploymentAffinityCacheValue(TypedDict):
-    model_id: str
+class DeploymentAffinityCacheValue(TypedDict, closed=True):
+    model_id: ReadOnly[str]
 
 
 VALID_MODEL_GROUP_AFFINITY_FLAGS: Final = frozenset(
@@ -60,19 +60,6 @@ def warn_on_unknown_model_group_affinity_flags(model_group_affinity_config: Mapp
             )
 
 
-_CLAIM_PIN_SCRIPT: Final = """
-local current = redis.call('GET', KEYS[1])
-if current == false then
-  redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
-  return ARGV[1]
-end
-if current == ARGV[1] then
-  redis.call('EXPIRE', KEYS[1], ARGV[2])
-end
-return current
-"""
-
-
 class DeploymentAffinityCheck(CustomLogger):
     """
     Router deployment affinity callback.
@@ -255,34 +242,33 @@ class DeploymentAffinityCheck(CustomLogger):
         return f"{cls.CACHE_KEY_PREFIX}:session:{model_group}:{hashed_user_key}:{session_id}"
 
     @staticmethod
-    def _get_session_id_from_metadata_dict(metadata: dict) -> str | None:
+    def _get_session_id_from_metadata_dict(metadata: Mapping[object, object]) -> str | None:
         session_id: Final = metadata.get("session_id")
         if session_id is None or metadata.get(SESSION_ID_GENERATED_METADATA_KEY):
             return None
         return str(session_id)
 
     @staticmethod
-    def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]:
+    def _iter_metadata_dicts(request_kwargs: Mapping[str, object]) -> tuple[Mapping[object, object], ...]:
         """
         Return all metadata dicts available on the request.
 
         Depending on the endpoint, Router may populate `metadata` or `litellm_metadata`.
         Users may also send one or both, so we check both (rather than using `or`).
         """
-        metadata_dicts: Final[list[dict]] = []
-        for key in ("litellm_metadata", "metadata"):
-            md = request_kwargs.get(key)
-            if isinstance(md, dict):
-                metadata_dicts.append(md)
-        return metadata_dicts
+        return tuple(
+            cast(Mapping[object, object], metadata)  # cast-ok: isinstance proves mapping shape; values remain opaque
+            for key in ("litellm_metadata", "metadata")
+            if isinstance(metadata := request_kwargs.get(key), dict)
+        )
 
     @staticmethod
-    def _first_metadata_value(metadata_dicts: Sequence[dict], key: str) -> str | None:
+    def _first_metadata_value(metadata_dicts: Sequence[Mapping[object, object]], key: str) -> str | None:
         value: Final = next((metadata[key] for metadata in metadata_dicts if metadata.get(key) is not None), None)
         return None if value is None else str(value)
 
     @classmethod
-    def _get_user_key_from_request_kwargs(cls, request_kwargs: dict) -> str | None:
+    def get_user_key_from_request_kwargs(cls, request_kwargs: Mapping[str, object]) -> str | None:
         """
         Extract a stable affinity key from request kwargs.
 
@@ -334,74 +320,17 @@ class DeploymentAffinityCheck(CustomLogger):
         return None
 
     def _set_local_pin(self, cache_key: str, value: object, ttl_seconds: int) -> None:
-        """The one owner of authoritative local pin writes: a plain set keeps a live
-        key's original expiry (`allow_ttl_override`), so the entry is replaced to make
-        the TTL real. Every local pin write goes through here so the redis-winner sync
-        and the pod-local claim can never disagree about expiry again."""
-        self.cache.in_memory_cache.delete_cache(cache_key)
-        self.cache.in_memory_cache.set_cache(cache_key, value, ttl=ttl_seconds)
+        set_local_affinity_pin(self.cache, cache_key, value, ttl_seconds)
 
     async def _claim_pin(self, cache_key: str, pin_value: DeploymentAffinityCacheValue, ttl_seconds: int) -> str | None:
-        """First-writer-wins pin write: store `pin_value` only when the key is absent and
-        return the deployment id the key holds afterwards, so a caller learns whether it won
-        by comparing against its own id, and None when the stored value is one no reader can
-        interpret. Concurrent claimers converge on the
-        first write instead of the last. Re-claiming with the stored value refreshes its
-        TTL, the same keepalive the complexity router's model pin documents: an active
-        session must not lose its pin mid-conversation just because it outlives the
-        original write, so the affinity TTL (the Router's
-        `deployment_affinity_ttl_seconds`, or a pre-routing hook's per-request
-        `session_affinity_ttl_seconds` override) bounds idle time, not total
-        session length. On Redis one Lua script does the get-or-set-or-refresh
-        atomically (same registration seam the rate limiters use) and the in-memory
-        tier is synchronized to the winner; without Redis, and whenever Redis is
-        unreachable, the pod-local check-and-set below stands in and is atomic because it
-        runs synchronously on the event loop. Degrading to a pod-local claim rather than
-        propagating the fault is what keeps same-pod stickiness through a Redis blip: the
-        caller only logs this result, so an escaping error would leave the session with no
-        pin at all and reshuffle every turn for the outage, which is worse than losing
-        cross-pod agreement. The redis tier is
-        resolved per call because the proxy attaches it after Router construction
-        (`Router._update_redis_cache`); the compiled script is cached per event loop
-        underneath the registration seam.
-        """
-        redis_cache: Final = self.cache.redis_cache
-        if redis_cache is not None:
-            try:
-                claim_script: Final = redis_cache.async_register_script(_CLAIM_PIN_SCRIPT)
-                raw: Final = await claim_script(keys=(cache_key,), args=(json.dumps(pin_value), int(ttl_seconds)))
-                decoded: Final = raw.decode("utf-8") if isinstance(raw, bytes) else raw
-                if not isinstance(decoded, str):
-                    return pin_value["model_id"]
-                try:
-                    winner: object = json.loads(decoded)
-                except json.JSONDecodeError:
-                    winner = decoded
-                self._set_local_pin(cache_key=cache_key, value=winner, ttl_seconds=ttl_seconds)
-                return self._pinned_model_id(winner)
-            except Exception as e:  # noqa: BLE001  # any Redis/Lua failure degrades to the pod-local claim, never unpins
-                verbose_router_logger.debug(
-                    "DeploymentAffinityCheck: redis pin claim failed, falling back to pod-local claim. error=%s", e
-                )
-
-        return self._claim_pin_in_memory(cache_key=cache_key, pin_value=pin_value, ttl_seconds=ttl_seconds)
+        winner: Final = await claim_affinity_pin(self.cache, cache_key, pin_value, ttl_seconds)
+        return self._pinned_model_id(winner)
 
     def _claim_pin_in_memory(
         self, cache_key: str, pin_value: DeploymentAffinityCacheValue, ttl_seconds: int
     ) -> str | None:
-        """Pod-local half of the claim, used when no Redis tier is attached and as the
-        fallback when the Redis claim fails. Mirrors the Lua script exactly, including
-        the keepalive: re-claiming with the stored value slides the idle window through
-        `_set_local_pin`. Both branches stay synchronous, hence atomic on the event
-        loop."""
-        existing: Final = self.cache.in_memory_cache.get_cache(cache_key)
-        if existing is not None:
-            existing_model_id: Final = self._pinned_model_id(existing)
-            if existing_model_id == pin_value["model_id"]:
-                self._set_local_pin(cache_key=cache_key, value=pin_value, ttl_seconds=ttl_seconds)
-            return existing_model_id
-        self._set_local_pin(cache_key=cache_key, value=pin_value, ttl_seconds=ttl_seconds)
-        return pin_value["model_id"]
+        winner: Final = claim_affinity_pin_in_memory(self.cache, cache_key, pin_value, ttl_seconds)
+        return self._pinned_model_id(winner)
 
     @staticmethod
     def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
@@ -465,7 +394,7 @@ class DeploymentAffinityCheck(CustomLogger):
             enable_session_id or self._get_marker_session_affinity_ttl(request_kwargs=request_kwargs) is not None
         )
         user_key: Final = (
-            self._get_user_key_from_request_kwargs(request_kwargs=request_kwargs)
+            self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs)
             if (session_affinity_active or enable_user_key)
             else None
         )
@@ -580,7 +509,7 @@ class DeploymentAffinityCheck(CustomLogger):
             return None
 
         user_key: Final = (
-            self._get_user_key_from_request_kwargs(request_kwargs=kwargs)
+            self.get_user_key_from_request_kwargs(request_kwargs=kwargs)
             if (enable_user_key or session_affinity_active)
             else None
         )

litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py

@@ -48,6 +48,7 @@ from litellm.exceptions import (
     ServiceUnavailableError,
 )
 from litellm.integrations.custom_logger import CustomLogger, Span
+from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     encrypted_content_of_block,
     strip_encrypted_reasoning_from_messages,
@@ -215,11 +216,11 @@ class EncryptedContentAffinityCheck(CustomLogger):
     @staticmethod
     def _encryption_boundary_key(
         litellm_params: object,
-    ) -> tuple | None:
+    ) -> tuple[object, object] | None:
         """
-        ``(api_base, api_key)`` pair identifying an Azure resource. Two
-        deployments sharing both are interchangeable for ``encrypted_content``
-        follow-ups; Azure rejects content produced by any other resource.
+        ``(api_base, api_key)`` identifies an upstream encryption boundary.
+        The values are resolved from the deployment and its named credential
+        without modifying the deployment.
 
         Accepts any object exposing dict-style ``.get(key, default)``: plain
         dicts (the common case in ``healthy_deployments``) as well as
@@ -234,9 +235,25 @@ class EncryptedContentAffinityCheck(CustomLogger):
             return None
         api_base: Final = getter("api_base")
         api_key: Final = getter("api_key")
-        if not api_base or not api_key:
+        credential_name: Final = getter("litellm_credential_name")
+        credential_values: Final[Mapping[str, object] | None] = (
+            CredentialAccessor.get_credential_values(credential_name)
+            if isinstance(credential_name, str) and credential_name
+            else None
+        )
+        effective_api_base: Final = (
+            credential_values.get("api_base")
+            if credential_values is not None and "api_base" in credential_values
+            else api_base
+        )
+        effective_api_key: Final = (
+            credential_values.get("api_key")
+            if credential_values is not None and "api_key" in credential_values
+            else api_key
+        )
+        if not effective_api_base or not effective_api_key:
             return None
-        return (api_base, api_key)
+        return (effective_api_base, effective_api_key)
 
     def _find_deployments_on_same_encryption_boundary(
         self,

litellm/rust_bridge/_native.pyi

@@ -0,0 +1,161 @@
+from asyncio import Future
+from collections.abc import Coroutine, Mapping, Sequence
+from typing import Literal, Never, TypeAlias, final
+
+from litellm.llms.base_llm.ocr.transformation import OCRResponse
+from litellm.rust_bridge.ocr import LiteLLMOcrRequest
+
+_InputSource: TypeAlias = Literal["request", "deployment", "environment"]
+
+class RustBridgeDeclined(Exception): ...
+class RustUpstreamError(Exception): ...
+
+def ocr(
+    model: str,
+    document: object,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    optional_params: Mapping[str, object] | None = None,
+    input_sources: Mapping[str, _InputSource] | None = None,
+    timeout_seconds: float | None = None,
+) -> dict[str, object]: ...
+def aocr(
+    model: str,
+    document: object,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    optional_params: Mapping[str, object] | None = None,
+    input_sources: Mapping[str, _InputSource] | None = None,
+    timeout_seconds: float | None = None,
+) -> Future[dict[str, object]]: ...
+
+_OCR_MAX_FILE_BYTES: int
+
+def _ocr_upload_document(
+    file_content: bytes,
+    file_name: str | None = None,
+    content_type: str | None = None,
+) -> dict[str, str]: ...
+def _ocr_file_document(document: Mapping[str, object]) -> dict[str, str]: ...
+def _ocr_mime_type(file_name: str) -> str: ...
+def _ocr_lifecycle(
+    request: LiteLLMOcrRequest,
+    args: tuple[object, ...],
+    kwargs: dict[str, object],
+    asynchronous: bool,
+) -> OCRResponse | Coroutine[object, object, OCRResponse]: ...
+def transcription(
+    model: str,
+    audio: object,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    optional_params: Mapping[str, object] | None = None,
+    timeout_seconds: float | None = None,
+) -> dict[str, object]: ...
+def atranscription(
+    model: str,
+    audio: object,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    optional_params: Mapping[str, object] | None = None,
+    timeout_seconds: float | None = None,
+) -> Future[dict[str, object]]: ...
+def messages(
+    model: str,
+    body: Mapping[str, object],
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    timeout_seconds: float | None = None,
+) -> dict[str, object]: ...
+def amessages(
+    model: str,
+    body: Mapping[str, object],
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    timeout_seconds: float | None = None,
+) -> Future[dict[str, object]]: ...
+def chat_completions_decline(
+    model: str,
+    messages: Sequence[object],
+    optional_params: Mapping[str, object] | None = None,
+    custom_llm_provider: str | None = None,
+) -> str | None: ...
+def chat_completions(
+    model: str,
+    messages: Sequence[object],
+    optional_params: Mapping[str, object] | None = None,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    timeout_seconds: float | None = None,
+) -> dict[str, object]: ...
+def achat_completions(
+    model: str,
+    messages: Sequence[object],
+    optional_params: Mapping[str, object] | None = None,
+    api_key: str | None = None,
+    api_base: str | None = None,
+    custom_llm_provider: str | None = None,
+    extra_headers: Mapping[str, object] | None = None,
+    timeout_seconds: float | None = None,
+) -> Future[dict[str, object]]: ...
+
+@final
+class ResponsesWebSocketConnection:
+    def __new__(cls, _uninstantiable: Never, /) -> Never: ...
+    @classmethod
+    def connect(
+        cls,
+        url: str,
+        headers: Mapping[str, str] | None = None,
+        timeout_seconds: float | None = None,
+    ) -> Future[ResponsesWebSocketConnection]: ...
+    def send_text(self, text: str) -> Future[None]: ...
+    def recv_text(self) -> Future[str | None]: ...
+    def close(self) -> Future[None]: ...
+
+@final
+class TokenCounter:
+    def __new__(cls, tokenizer_json: str) -> TokenCounter: ...
+    @staticmethod
+    def from_cl100k_ranks(rank_file: str) -> TokenCounter: ...
+    @staticmethod
+    def from_o200k_ranks(rank_file: str) -> TokenCounter: ...
+    def acount_request(self, body: bytes) -> Future[dict[str, object]]: ...
+
+def gil_stats() -> dict[str, int]: ...
+
+__all__ = [
+    "_OCR_MAX_FILE_BYTES",
+    "ResponsesWebSocketConnection",
+    "RustBridgeDeclined",
+    "RustUpstreamError",
+    "TokenCounter",
+    "_ocr_file_document",
+    "_ocr_lifecycle",
+    "_ocr_mime_type",
+    "_ocr_upload_document",
+    "achat_completions",
+    "amessages",
+    "aocr",
+    "atranscription",
+    "chat_completions",
+    "chat_completions_decline",
+    "gil_stats",
+    "messages",
+    "ocr",
+    "transcription",
+]

litellm/types/guardrails.py

@@ -209,6 +209,15 @@ class PiiEntityCategory(str, Enum):
     AUSTRALIA = "Australia"
     INDIA = "India"
     FINLAND = "Finland"
+    GERMANY = "Germany"
+    KOREA = "Korea"
+    CANADA = "Canada"
+    SWEDEN = "Sweden"
+    THAILAND = "Thailand"
+    TURKEY = "Turkey"
+    NIGERIA = "Nigeria"
+    PHILIPPINES = "Philippines"
+    SOUTH_AFRICA = "South Africa"
 
 
 class PiiEntityType(str, Enum):
@@ -225,21 +234,27 @@ class PiiEntityType(str, Enum):
     PHONE_NUMBER = "PHONE_NUMBER"
     MEDICAL_LICENSE = "MEDICAL_LICENSE"
     URL = "URL"
+    MAC_ADDRESS = "MAC_ADDRESS"
+    UUID = "UUID"
     # USA
     US_BANK_NUMBER = "US_BANK_NUMBER"
     US_DRIVER_LICENSE = "US_DRIVER_LICENSE"
     US_ITIN = "US_ITIN"
     US_PASSPORT = "US_PASSPORT"
     US_SSN = "US_SSN"
+    US_MBI = "US_MBI"
+    US_NPI = "US_NPI"
     # UK
     UK_NHS = "UK_NHS"
     UK_NINO = "UK_NINO"
     UK_PASSPORT = "UK_PASSPORT"
     UK_POSTCODE = "UK_POSTCODE"
     UK_VEHICLE_REGISTRATION = "UK_VEHICLE_REGISTRATION"
+    UK_DRIVING_LICENCE = "UK_DRIVING_LICENCE"
     # Spain
     ES_NIF = "ES_NIF"
     ES_NIE = "ES_NIE"
+    ES_PASSPORT = "ES_PASSPORT"
     # Italy
     IT_FISCAL_CODE = "IT_FISCAL_CODE"
     IT_DRIVER_LICENSE = "IT_DRIVER_LICENSE"
@@ -262,13 +277,53 @@ class PiiEntityType(str, Enum):
     IN_VEHICLE_REGISTRATION = "IN_VEHICLE_REGISTRATION"
     IN_VOTER = "IN_VOTER"
     IN_PASSPORT = "IN_PASSPORT"
+    IN_GSTIN = "IN_GSTIN"
     # Finland
     FI_PERSONAL_IDENTITY_CODE = "FI_PERSONAL_IDENTITY_CODE"
+    # Germany
+    DE_TAX_ID = "DE_TAX_ID"
+    DE_TAX_NUMBER = "DE_TAX_NUMBER"
+    DE_VAT_ID = "DE_VAT_ID"
+    DE_PASSPORT = "DE_PASSPORT"
+    DE_ID_CARD = "DE_ID_CARD"
+    DE_FUEHRERSCHEIN = "DE_FUEHRERSCHEIN"
+    DE_SOCIAL_SECURITY = "DE_SOCIAL_SECURITY"
+    DE_HEALTH_INSURANCE = "DE_HEALTH_INSURANCE"
+    DE_LANR = "DE_LANR"
+    DE_BSNR = "DE_BSNR"
+    DE_KFZ = "DE_KFZ"
+    DE_HANDELSREGISTER = "DE_HANDELSREGISTER"
+    DE_PLZ = "DE_PLZ"
+    # Korea
+    KR_RRN = "KR_RRN"
+    KR_FRN = "KR_FRN"
+    KR_PASSPORT = "KR_PASSPORT"
+    KR_DRIVER_LICENSE = "KR_DRIVER_LICENSE"
+    KR_BRN = "KR_BRN"
+    # Canada
+    CA_SIN = "CA_SIN"
+    # Sweden
+    SE_PERSONNUMMER = "SE_PERSONNUMMER"
+    SE_ORGANISATIONSNUMMER = "SE_ORGANISATIONSNUMMER"
+    # Thailand
+    TH_TNIN = "TH_TNIN"
+    # Turkey
+    TR_NATIONAL_ID = "TR_NATIONAL_ID"
+    TR_LICENSE_PLATE = "TR_LICENSE_PLATE"
+    # Nigeria
+    NG_NIN = "NG_NIN"
+    NG_VEHICLE_REGISTRATION = "NG_VEHICLE_REGISTRATION"
+    # Philippines
+    PH_TIN = "PH_TIN"
+    PH_UMID = "PH_UMID"
+    PH_PASSPORT = "PH_PASSPORT"
+    # South Africa
+    ZA_ID_NUMBER = "ZA_ID_NUMBER"
 
 
 # Define mappings of PII entity types by category
 PII_ENTITY_CATEGORIES_MAP: Final = {
-    PiiEntityCategory.GENERAL: [
+    PiiEntityCategory.GENERAL: (
         PiiEntityType.DATE_TIME,
         PiiEntityType.EMAIL_ADDRESS,
         PiiEntityType.IP_ADDRESS,
@@ -278,50 +333,85 @@ PII_ENTITY_CATEGORIES_MAP: Final = {
         PiiEntityType.PHONE_NUMBER,
         PiiEntityType.MEDICAL_LICENSE,
         PiiEntityType.URL,
-    ],
-    PiiEntityCategory.FINANCE: [
+        PiiEntityType.MAC_ADDRESS,
+        PiiEntityType.UUID,
+    ),
+    PiiEntityCategory.FINANCE: (
         PiiEntityType.CREDIT_CARD,
         PiiEntityType.CRYPTO,
         PiiEntityType.IBAN_CODE,
-    ],
-    PiiEntityCategory.USA: [
+    ),
+    PiiEntityCategory.USA: (
         PiiEntityType.US_BANK_NUMBER,
         PiiEntityType.US_DRIVER_LICENSE,
         PiiEntityType.US_ITIN,
         PiiEntityType.US_PASSPORT,
         PiiEntityType.US_SSN,
-    ],
-    PiiEntityCategory.UK: [
+        PiiEntityType.US_MBI,
+        PiiEntityType.US_NPI,
+    ),
+    PiiEntityCategory.UK: (
         PiiEntityType.UK_NHS,
         PiiEntityType.UK_NINO,
         PiiEntityType.UK_PASSPORT,
         PiiEntityType.UK_POSTCODE,
         PiiEntityType.UK_VEHICLE_REGISTRATION,
-    ],
-    PiiEntityCategory.SPAIN: [PiiEntityType.ES_NIF, PiiEntityType.ES_NIE],
-    PiiEntityCategory.ITALY: [
+        PiiEntityType.UK_DRIVING_LICENCE,
+    ),
+    PiiEntityCategory.SPAIN: (PiiEntityType.ES_NIF, PiiEntityType.ES_NIE, PiiEntityType.ES_PASSPORT),
+    PiiEntityCategory.ITALY: (
         PiiEntityType.IT_FISCAL_CODE,
         PiiEntityType.IT_DRIVER_LICENSE,
         PiiEntityType.IT_VAT_CODE,
         PiiEntityType.IT_PASSPORT,
         PiiEntityType.IT_IDENTITY_CARD,
-    ],
-    PiiEntityCategory.POLAND: [PiiEntityType.PL_PESEL],
-    PiiEntityCategory.SINGAPORE: [PiiEntityType.SG_NRIC_FIN, PiiEntityType.SG_UEN],
-    PiiEntityCategory.AUSTRALIA: [
+    ),
+    PiiEntityCategory.POLAND: (PiiEntityType.PL_PESEL,),
+    PiiEntityCategory.SINGAPORE: (PiiEntityType.SG_NRIC_FIN, PiiEntityType.SG_UEN),
+    PiiEntityCategory.AUSTRALIA: (
         PiiEntityType.AU_ABN,
         PiiEntityType.AU_ACN,
         PiiEntityType.AU_TFN,
         PiiEntityType.AU_MEDICARE,
-    ],
-    PiiEntityCategory.INDIA: [
+    ),
+    PiiEntityCategory.INDIA: (
         PiiEntityType.IN_PAN,
         PiiEntityType.IN_AADHAAR,
         PiiEntityType.IN_VEHICLE_REGISTRATION,
         PiiEntityType.IN_VOTER,
         PiiEntityType.IN_PASSPORT,
-    ],
-    PiiEntityCategory.FINLAND: [PiiEntityType.FI_PERSONAL_IDENTITY_CODE],
+        PiiEntityType.IN_GSTIN,
+    ),
+    PiiEntityCategory.FINLAND: (PiiEntityType.FI_PERSONAL_IDENTITY_CODE,),
+    PiiEntityCategory.GERMANY: (
+        PiiEntityType.DE_TAX_ID,
+        PiiEntityType.DE_TAX_NUMBER,
+        PiiEntityType.DE_VAT_ID,
+        PiiEntityType.DE_PASSPORT,
+        PiiEntityType.DE_ID_CARD,
+        PiiEntityType.DE_FUEHRERSCHEIN,
+        PiiEntityType.DE_SOCIAL_SECURITY,
+        PiiEntityType.DE_HEALTH_INSURANCE,
+        PiiEntityType.DE_LANR,
+        PiiEntityType.DE_BSNR,
+        PiiEntityType.DE_KFZ,
+        PiiEntityType.DE_HANDELSREGISTER,
+        PiiEntityType.DE_PLZ,
+    ),
+    PiiEntityCategory.KOREA: (
+        PiiEntityType.KR_RRN,
+        PiiEntityType.KR_FRN,
+        PiiEntityType.KR_PASSPORT,
+        PiiEntityType.KR_DRIVER_LICENSE,
+        PiiEntityType.KR_BRN,
+    ),
+    PiiEntityCategory.CANADA: (PiiEntityType.CA_SIN,),
+    PiiEntityCategory.SWEDEN: (PiiEntityType.SE_PERSONNUMMER, PiiEntityType.SE_ORGANISATIONSNUMMER),
+    PiiEntityCategory.THAILAND: (PiiEntityType.TH_TNIN,),
+    PiiEntityCategory.TURKEY: (PiiEntityType.TR_NATIONAL_ID, PiiEntityType.TR_LICENSE_PLATE),
+    PiiEntityCategory.NIGERIA: (PiiEntityType.NG_NIN, PiiEntityType.NG_VEHICLE_REGISTRATION),
+    PiiEntityCategory.PHILIPPINES: (PiiEntityType.PH_TIN, PiiEntityType.PH_UMID, PiiEntityType.PH_PASSPORT),
+    PiiEntityCategory.SOUTH_AFRICA: (PiiEntityType.ZA_ID_NUMBER,),
 }
 
 

litellm/types/llms/anthropic.py

@@ -746,6 +746,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum):
     ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20"
     FAST_MODE_2026_02_01 = "fast-mode-2026-02-01"
     ADVISOR_TOOL_2026_03_01 = "advisor-tool-2026-03-01"
+    PER_TURN_CONTROL_2026_07_01 = "per-turn-control-2026-07-01"
 
 
 # Tool search beta header constant (for Anthropic direct API and Microsoft Foundry)