Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_enterprise_request_metering

.circleci/config.yml

@@ -1029,6 +1029,8 @@ jobs:
       - *python312_image
     working_directory: ~/project
     resource_class: large
+    environment:
+      REQUEST_TIMEOUT: "180"
 
     steps:
       - checkout
@@ -1058,7 +1060,8 @@ jobs:
                 -v -x \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
-                -n 8"
+                -n 8 \
+                --reruns 1 --only-rerun Timeout"
           no_output_timeout: 15m
 
       # Store test results

.github/workflows/codspeed.yml

@@ -4,9 +4,11 @@ on:
   push:
     branches:
       - main
+      - litellm_internal_staging
   pull_request:
     branches:
       - main
+      - litellm_internal_staging
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -22,7 +24,7 @@ concurrency:
 jobs:
   benchmarks:
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 60
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0

.github/workflows/test-terraform-provider.yml

@@ -0,0 +1,113 @@
+name: Terraform Provider
+
+on:
+  push:
+    paths:
+      - "terraform/provider/**"
+      - ".github/workflows/test-terraform-provider.yml"
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "terraform/provider/**"
+      - "litellm/proxy/**"
+      - ".github/workflows/test-terraform-provider.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  provider-checks:
+    name: gofmt, vet, build, test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: terraform/provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
+        with:
+          go-version-file: terraform/provider/go.mod
+          cache: true
+          cache-dependency-path: terraform/provider/go.sum
+
+      - name: gofmt
+        run: |
+          UNFORMATTED=$(gofmt -l .)
+          if [ -n "${UNFORMATTED}" ]; then
+            echo "::error::gofmt required for: ${UNFORMATTED}"
+            exit 1
+          fi
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: Build
+        run: go build ./...
+
+      - name: Test
+        run: go test -timeout 120s ./...
+
+  endpoint-drift:
+    name: Provider endpoints vs proxy OpenAPI schema
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Generate proxy OpenAPI schema
+        run: |
+          uv run --no-sync python terraform/provider/tools/dump_openapi.py "${RUNNER_TEMP}/openapi.json"
+
+      - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
+        with:
+          go-version-file: terraform/provider/go.mod
+          cache: true
+          cache-dependency-path: terraform/provider/go.sum
+
+      - name: Audit provider endpoints against the schema
+        working-directory: terraform/provider
+        run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"

.github/workflows/test_server_root_path.yml

@@ -16,6 +16,7 @@ jobs:
     timeout-minutes: 30
 
     strategy:
+      fail-fast: false
       matrix:
         root_path: ["/api/v1", "/llmproxy"]
 
@@ -108,8 +109,26 @@ jobs:
       - name: Install UI deps and Chromium
         working-directory: ui/litellm-dashboard
         run: |
-          npm ci
-          npx playwright install --with-deps chromium
+          retry() {
+            local attempt=1
+            local max_attempts=4
+            until "$@"; do
+              if [ "$attempt" -ge "$max_attempts" ]; then
+                echo "Command failed after $attempt attempts: $*"
+                return 1
+              fi
+              echo "Attempt $attempt failed: $*. Retrying in $((attempt * 15))s..."
+              sleep $((attempt * 15))
+              attempt=$((attempt + 1))
+            done
+          }
+
+          npm config set fetch-retries 5
+          npm config set fetch-retry-mintimeout 20000
+          npm config set fetch-retry-maxtimeout 120000
+
+          retry npm ci
+          retry npx playwright install --with-deps chromium
 
       - name: Run SERVER_ROOT_PATH redirect e2e
         working-directory: ui/litellm-dashboard

CLAUDE.md

@@ -21,11 +21,11 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
 
 When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
 
-When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout
+When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
 
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
+Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
 
 If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
 - don't use emojis

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -919,9 +919,9 @@ class BaseEmailLogger(CustomLogger):
         """
         Construct invitation link for the user
 
-        # http://localhost:4000/ui?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b
+        # http://localhost:4000/ui/onboarding?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b
         """
-        return f"{base_url}/ui?invitation_id={invitation_id}"
+        return f"{base_url}/ui/onboarding?invitation_id={invitation_id}"
 
     async def send_email(
         self,

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.47"
+version = "0.1.48"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.47"
+version = "0.1.48"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/main.py

@@ -25,17 +25,25 @@ DatabaseURLSettings.from_env().apply_to_env()
 
 from litellm.proxy.proxy_server import app
 
-from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES
+from gateway.routes.allowlist import (
+    GATEWAY_EXACT_PATHS,
+    GATEWAY_MOUNT_PATHS,
+    GATEWAY_PATH_PREFIXES,
+)
 
 
 def _is_gateway_route(route) -> bool:
-    """Keep the route on the gateway if its path is in the LLM data-plane surface."""
+    """Keep the route on the gateway if its path is in the LLM data-plane surface.
+
+    Prometheus registers /metrics as a Mount (``app.mount("/metrics", make_asgi_app())``),
+    so Mounts are matched against GATEWAY_MOUNT_PATHS instead of being dropped with
+    the UI static mounts.
+    """
     path = getattr(route, "path", None)
     if path is None:
         return False
     if isinstance(route, Mount):
-        # Gateway never serves the static UI or its asset bundles.
-        return False
+        return path in GATEWAY_MOUNT_PATHS
     if path in GATEWAY_EXACT_PATHS:
         return True
     return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES)

gateway/routes/allowlist.py

@@ -106,7 +106,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     # Health & ops
     "/health",
     "/metrics",
-    "/watsonx"
+    "/watsonx",
 )
 
 GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
@@ -120,3 +120,9 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
         "/test",
     }
 )
+
+GATEWAY_MOUNT_PATHS: frozenset[str] = frozenset(
+    {
+        "/metrics",
+    }
+)

litellm-proxy-extras/litellm_proxy_extras/migrations/20260630120000_add_token_exchange_to_mcp_servers/migration.sql

@@ -0,0 +1,8 @@
+-- Timestamp sorts before some already-applied migrations; this is safe: the
+-- runner is `prisma migrate deploy`, which applies every pending migration
+-- regardless of name order (utils.py has an informational check for exactly
+-- this), and IF NOT EXISTS keeps a re-apply idempotent.
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "token_exchange_endpoint" TEXT;
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "audience" TEXT;
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "subject_token_type" TEXT;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260703120000_add_token_exchange_profile_to_mcp_servers/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "token_exchange_profile" TEXT;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -329,6 +329,12 @@ model LiteLLM_MCPServerTable {
   token_url          String?
   registration_url   String?
   oauth2_flow        String?
+  token_exchange_endpoint String?
+  // Named for the RFC 8693 "audience" token-exchange request parameter (that flow only).
+  // RFC 8707 resource indicators are a separate concept, named "resource" in the v2 egress types.
+  audience           String?
+  subject_token_type String?
+  token_exchange_profile String?
   allow_all_keys Boolean @default(false)
   available_on_public_internet Boolean @default(true)
   delegate_auth_to_upstream Boolean @default(false)

litellm/__init__.py

@@ -379,6 +379,7 @@ budget_duration: Optional[str] = (
     None  # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
 )
 default_soft_budget: float = DEFAULT_SOFT_BUDGET  # by default all litellm proxy keys have a soft budget of 50.0
+budget_exceeded_throttle_percentage: Optional[float] = None
 forward_traceparent_to_llm_provider: bool = False
 
 

litellm/a2a_protocol/streaming_iterator.py

@@ -11,7 +11,6 @@ from litellm._logging import verbose_logger
 from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
 from litellm.a2a_protocol.utils import A2ARequestUtils
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.litellm_core_utils.thread_pool_executor import executor
 
 if TYPE_CHECKING:
     from a2a.types import SendStreamingMessageRequest, SendStreamingMessageResponse
@@ -128,22 +127,15 @@ class A2AStreamingIterator:
 
             # Call success handlers - they will build standard_logging_object
             asyncio.create_task(
-                self.logging_obj.async_success_handler(
-                    result=result,
+                self.logging_obj.dispatch_success_handlers(
+                    result,
                     start_time=self.start_time,
                     end_time=end_time,
                     cache_hit=None,
+                    prefer_async_handlers=True,
                 )
             )
 
-            executor.submit(
-                self.logging_obj.success_handler,
-                result=result,
-                cache_hit=None,
-                start_time=self.start_time,
-                end_time=end_time,
-            )
-
             verbose_logger.info(
                 f"A2A streaming completed: prompt_tokens={prompt_tokens}, "
                 f"completion_tokens={completion_tokens}, total_tokens={total_tokens}, "

litellm/constants.py

@@ -1504,6 +1504,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
     "public_model_groups_links",
     "cost_discount_config",
     "cost_margin_config",
+    "budget_exceeded_throttle_percentage",
 ]
 SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
 DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))

litellm/integrations/otel/__init__.py

@@ -52,6 +52,7 @@ from litellm.integrations.otel.model.semconv import (
     GenAIProvider,
     JsonRpc,
     LiteLLM,
+    LiteLLMError,
     MCPMethod,
     Metric,
     Network,
@@ -87,6 +88,7 @@ __all__ = [
     "HTTP",
     "JsonRpc",
     "LiteLLM",
+    "LiteLLMError",
     "MCP",
     "MCPMethod",
     "Metric",

litellm/integrations/otel/emitter.py

@@ -16,9 +16,10 @@ from litellm.integrations.otel.model.payloads import (
     MCPListToolsSpanData,
     MCPToolCallSpanData,
     ServiceSpanData,
+    SpanError,
 )
 from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
-from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
+from litellm.integrations.otel.model.semconv import Error, ExceptionEvent, LiteLLMError
 from litellm.integrations.otel.model.spans import (
     SPAN_REGISTRY,
     SpanRole,
@@ -49,6 +50,27 @@ _NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = {
 _DEDUP_CACHE_MAX = 10_000
 
 
+def _stamp_otel_error_attributes(span: Span, error_type: str, resolved_message: str) -> None:
+    """Stamp the OTel-semconv error attributes (``error.type`` + ``error.message``).
+    ``error_type`` and ``resolved_message`` are ``finish_span``'s already-computed
+    fallback chains, so the pair on the status, event, and attributes stays in
+    lockstep."""
+    span.set_attribute(Error.TYPE, error_type)
+    span.set_attribute(Error.MESSAGE, resolved_message)
+
+
+def _stamp_litellm_error_attributes(span: Span, error: SpanError) -> None:
+    """Stamp litellm-specific error detail attributes. Emitted only when the
+    corresponding field is populated so guardrail-shape errors carrying only a
+    message aren't polluted with empty detail keys."""
+    if error.code:
+        span.set_attribute(LiteLLMError.CODE, error.code)
+    if error.stack_trace:
+        span.set_attribute(LiteLLMError.STACK_TRACE, error.stack_trace)
+    if error.llm_provider:
+        span.set_attribute(LiteLLMError.LLM_PROVIDER, error.llm_provider)
+
+
 class SpanEmitter:
     def __init__(
         self,
@@ -190,12 +212,13 @@ class SpanEmitter:
         if error and (error.error_type or error.message):
             error_type = error.error_type or "error"
             message = error.message or error.error_type or "error"
-            span.set_attribute(Error.TYPE, error_type)
+            _stamp_otel_error_attributes(span, error_type, message)
+            _stamp_litellm_error_attributes(span, error)
             span.set_status(Status(StatusCode.ERROR, message))
-            # Carry the full message on the standard ``exception`` event so backends
-            # map it as full text under ``exception.message``. Setting it as a bare
-            # string attribute instead lets backends like Elasticsearch dynamic-map
-            # it to a ``keyword`` capped at 1024 chars, truncating the message.
+            # Also emit the semconv ``exception`` event so backends that
+            # dynamic-map unknown string span attrs to ``keyword`` (e.g.
+            # Elasticsearch with a 1024-char ``ignore_above``) still see the
+            # full untruncated message on the recognized event field.
             span.add_event(
                 ExceptionEvent.NAME,
                 {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},

litellm/integrations/otel/logger.py

@@ -368,7 +368,11 @@ class OpenTelemetryV2(CustomLogger):
                 # it (named provisionally) so it isn't leaked as an open span.
                 carrier.span.end(end_time=to_ns(end_time))
             return None
-        data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=self.config.capture_span_content)
+        data = LLMCallSpanData.from_standard_logging_payload(
+            payload,
+            capture_content=self.config.capture_span_content,
+            time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
+        )
         end_time_ns = to_ns(end_time)
         if carrier.span is not None:
             # Born at the boundary: stamp attributes from the typed payload, set

litellm/integrations/otel/mappers/genai.py

@@ -55,6 +55,7 @@ class GenAIMapper:
         GenAI.RESPONSE_MODEL: lambda d: d.response_model,
         GenAI.RESPONSE_ID: lambda d: d.response_id,
         GenAI.RESPONSE_FINISH_REASONS: lambda d: list(d.finish_reasons) if d.finish_reasons else None,
+        GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds,
         GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
         GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
         Error.TYPE: lambda d: d.error.error_type if d.error else None,

litellm/integrations/otel/model/metadata.py

@@ -41,7 +41,7 @@ from typing import TYPE_CHECKING, Any, Mapping, cast
 
 from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
 from litellm.integrations.otel.model.semconv import resolve_operation
-from litellm.integrations.otel.model.utils import as_str
+from litellm.integrations.otel.model.utils import as_str, to_seconds
 
 if TYPE_CHECKING:
     from litellm.types.utils import StandardLoggingPayload
@@ -201,6 +201,7 @@ class LLMCallEvent:
     # span is renamed from the typed payload at close (``finish_span``); this only
     # needs to be reasonable for a span that never gets closed (a leak).
     provisional_span_name: str
+    time_to_first_chunk_seconds: float | None
 
     @classmethod
     def from_dict(cls, kwargs: Mapping[str, Any]) -> "LLMCallEvent":
@@ -214,9 +215,25 @@ class LLMCallEvent:
             dynamic_params=kwargs.get("standard_callback_dynamic_params"),
             is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
             provisional_span_name=f"{operation.value} {model}".strip(),
+            time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
         )
 
 
+def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
+    """Seconds from the upstream request being issued (``api_call_start_time``)
+    to the first streamed chunk (``completion_start_time``); ``None`` for
+    non-streaming calls, where ``completion_start_time`` is backfilled with the
+    end time and would not measure first-chunk latency."""
+    optional_params = cast(Mapping[str, Any], kwargs.get("optional_params") or {})
+    if not optional_params.get("stream"):
+        return None
+    api_call_start = to_seconds(kwargs.get("api_call_start_time"))
+    completion_start = to_seconds(kwargs.get("completion_start_time"))
+    if api_call_start is None or completion_start is None:
+        return None
+    return completion_start - api_call_start
+
+
 def _call_id(payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any]) -> str | None:
     """The call id from the payload (when closed) or the bare kwargs (at pre_call)."""
     if payload is not None:

litellm/integrations/otel/model/payloads.py

@@ -141,6 +141,9 @@ class LLMCost:
 class SpanError:
     error_type: str | None = None
     message: str | None = None
+    code: str | None = None
+    stack_trace: str | None = None
+    llm_provider: str | None = None
 
 
 @dataclass(frozen=True)
@@ -305,10 +308,14 @@ class LLMCallSpanData:
     messages_in: tuple[Mapping[str, object], ...] = ()
     choices_out: tuple[Mapping[str, object], ...] = ()
     system_fingerprint: str | None = None
+    time_to_first_chunk_seconds: float | None = None
 
     @classmethod
     def from_standard_logging_payload(
-        cls, payload: "StandardLoggingPayload", capture_content: bool = False
+        cls,
+        payload: "StandardLoggingPayload",
+        capture_content: bool = False,
+        time_to_first_chunk_seconds: float | None = None,
     ) -> "LLMCallSpanData":
         params = cast(Mapping[str, object], payload.get("model_parameters") or {})
         # The single parse of the request's metadata — the request-vs-provider
@@ -349,6 +356,7 @@ class LLMCallSpanData:
             messages_in=_dicts(payload.get("messages")) if capture_content else (),
             choices_out=choices_out if capture_content else (),
             system_fingerprint=as_str(response.get("system_fingerprint")),
+            time_to_first_chunk_seconds=time_to_first_chunk_seconds,
         )
 
 
@@ -566,6 +574,9 @@ def _parse_error(payload: "StandardLoggingPayload") -> SpanError | None:
     return SpanError(
         error_type=as_str(info.get("error_class")) or as_str(info.get("error_code")),
         message=as_str(info.get("error_message")) or as_str(payload.get("error_str")),
+        code=as_str(info.get("error_code")),
+        stack_trace=as_str(info.get("traceback")),
+        llm_provider=as_str(info.get("llm_provider")),
     )
 
 

litellm/integrations/otel/model/semconv.py

@@ -69,6 +69,7 @@ class GenAI:
     RESPONSE_ID: Final = "gen_ai.response.id"
     RESPONSE_MODEL: Final = "gen_ai.response.model"
     RESPONSE_FINISH_REASONS: Final = "gen_ai.response.finish_reasons"
+    RESPONSE_TIME_TO_FIRST_CHUNK: Final = "gen_ai.response.time_to_first_chunk"
     # usage
     USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
     USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"
@@ -143,7 +144,27 @@ class Client:
 
 
 class Error:
+    """OTel-defined error attribute keys, from the semconv ``error.*`` registry.
+    ``MESSAGE`` is marked *Deprecated* upstream in favor of domain-specific
+    error message keys plus ``exception.message`` on the exception event, but
+    is still defined and stamped by litellm's v1 integration; keeping it here
+    for byte-for-byte parity."""
+
     TYPE: Final = "error.type"
+    MESSAGE: Final = "error.message"
+
+
+class LiteLLMError:
+    """LiteLLM-specific error attribute keys. Emitted under the ``error.*``
+    namespace (not ``litellm.*``) for byte-for-byte compat with the v1
+    integration in ``opentelemetry.py``; consumers reading these keys on v1
+    spans read the same keys on v2 spans. OTel semconv does not define any of
+    these three, and per its extension rules a namespace may carry additional
+    vendor keys as long as they don't collide with defined names."""
+
+    CODE: Final = "error.code"
+    STACK_TRACE: Final = "error.stack_trace"
+    LLM_PROVIDER: Final = "error.llm_provider"
 
 
 class ExceptionEvent:

litellm/integrations/otel/plumbing/metrics.py

@@ -21,6 +21,7 @@ from litellm.integrations.opentelemetry import (
     _build_metric_attribute_filter,
     _resolve_metric_attribute_filter,
 )
+from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds
 from litellm.integrations.otel.model.semconv import Metric, resolve_operation
 from litellm.integrations.otel.model.utils import to_seconds
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -181,13 +182,10 @@ class GenAIMetricRecorder:
         self._metrics.token_usage.record(usage.get("completion_tokens", 0), attributes=out_attrs)
 
     def _record_time_to_first_token(self, kwargs: Mapping[str, Any], common_attrs: dict) -> None:
-        if not kwargs.get("optional_params", {}).get("stream", False):
+        time_to_first_chunk = time_to_first_chunk_seconds(kwargs)
+        if time_to_first_chunk is None:
             return
-        api_call_start = to_seconds(kwargs.get("api_call_start_time"))
-        completion_start = to_seconds(kwargs.get("completion_start_time"))
-        if api_call_start is None or completion_start is None:
-            return
-        self._metrics.time_to_first_token.record(completion_start - api_call_start, attributes=common_attrs)
+        self._metrics.time_to_first_token.record(time_to_first_chunk, attributes=common_attrs)
 
     def _record_time_per_output_token(
         self,

litellm/integrations/prometheus.py

@@ -3619,6 +3619,7 @@ class PrometheusLogger(CustomLogger):
                     hashed_token=user_api_key_dict.token,
                     prisma_client=prisma_client,
                     user_api_key_cache=user_api_key_cache,
+                    check_cache_only=True,
                 )
                 if key_object:
                     user_api_key_dict.budget_reset_at = key_object.budget_reset_at

litellm/interactions/streaming_iterator.py

@@ -174,22 +174,15 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
         logging_response = copy.deepcopy(self.completed_response)
 
         asyncio.create_task(
-            self.logging_obj.async_success_handler(
-                result=logging_response,
+            self.logging_obj.dispatch_success_handlers(
+                logging_response,
                 start_time=self.start_time,
                 end_time=datetime.now(),
                 cache_hit=None,
+                prefer_async_handlers=True,
             )
         )
 
-        executor.submit(
-            self.logging_obj.success_handler,
-            result=logging_response,
-            cache_hit=None,
-            start_time=self.start_time,
-            end_time=datetime.now(),
-        )
-
 
 class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
     """

litellm/litellm_core_utils/health_check_helpers.py

@@ -95,6 +95,17 @@ class HealthCheckHelpers:
         """
         import litellm
 
+        logging_obj = filtered_model_params.get("litellm_logging_obj")
+        if logging_obj is not None:
+            api_base = filtered_model_params.get("api_base")
+            logging_obj.update_from_kwargs(
+                kwargs=filtered_model_params,
+                model=filtered_model_params.get("model"),
+                user=None,
+                optional_params={},
+                litellm_params={"api_base": api_base} if api_base else None,
+            )
+
         if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS:
             return await litellm.alist_batches(**filtered_model_params)
         else:
@@ -188,6 +199,7 @@ class HealthCheckHelpers:
                 api_base=model_params.get("api_base", None),
                 api_key=model_params.get("api_key", None),
                 api_version=model_params.get("api_version", None),
+                model_params=model_params,
             ),
             "batch": lambda: HealthCheckHelpers._batch_health_check(
                 custom_llm_provider=custom_llm_provider,

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -3626,6 +3626,7 @@ class BedrockImageProcessor:
 
 def _convert_to_bedrock_tool_call_invoke(
     tool_calls: list,
+    model: Optional[str] = None,
 ) -> List[BedrockContentBlock]:
     """
     OpenAI tool invokes:
@@ -3701,7 +3702,13 @@ def _convert_to_bedrock_tool_call_invoke(
                             # cache_control applies to the whole original
                             # tool call; attach after the last split block.
                             if tool.get("cache_control", None) is not None:
-                                _parts_list.append(BedrockContentBlock(cachePoint=CachePointBlock(type="default")))
+                                _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                                    {"cache_control": tool["cache_control"]},
+                                    block_type="content_block",
+                                    model=model,
+                                )
+                                if _cache_point_block is not None:
+                                    _parts_list.append(_cache_point_block)
                             continue
                         # Fallback: no objects extracted — use empty dict.
                         arguments_dict = {}
@@ -3712,8 +3719,13 @@ def _convert_to_bedrock_tool_call_invoke(
 
                 # Check for cache_control and add a separate cachePoint block
                 if tool.get("cache_control", None) is not None:
-                    cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
-                    _parts_list.append(cache_point_block)
+                    cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                        {"cache_control": tool["cache_control"]},
+                        block_type="content_block",
+                        model=model,
+                    )
+                    if cache_point_block is not None:
+                        _parts_list.append(cache_point_block)
         return _parts_list
     except Exception as e:
         raise Exception(
@@ -4377,6 +4389,7 @@ class BedrockConverseMessagesProcessor:
                             _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
                                 message_block=cast(OpenAIMessageContentListBlock, element),
                                 block_type="content_block",
+                                model=model,
                             )
                             if _cache_point_block is not None:
                                 _parts.append(_cache_point_block)
@@ -4384,7 +4397,7 @@ class BedrockConverseMessagesProcessor:
                 elif message_block["content"] and isinstance(message_block["content"], str):
                     _part = BedrockContentBlock(text=messages[msg_i]["content"])
                     _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
-                        message_block, block_type="content_block"
+                        message_block, block_type="content_block", model=model
                     )
                     user_content.append(_part)
                     if _cache_point_block is not None:
@@ -4416,22 +4429,27 @@ class BedrockConverseMessagesProcessor:
                 tool_content.append(tool_call_result)
 
                 # Check if we need to add a separate cachePoint block
-                has_cache_control = False
+                tool_msg_cache_control = None
 
                 # Check for message-level cache_control
                 if current_message.get("cache_control", None) is not None:
-                    has_cache_control = True
+                    tool_msg_cache_control = current_message["cache_control"]
                 # Check for content-level cache_control in list content
                 elif isinstance(current_message.get("content"), list):
                     for content_element in current_message["content"]:
                         if isinstance(content_element, dict) and content_element.get("cache_control", None) is not None:
-                            has_cache_control = True
+                            tool_msg_cache_control = content_element["cache_control"]
                             break
 
                 # Add a separate cachePoint block if cache_control is present
-                if has_cache_control:
-                    cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
-                    tool_content.append(cache_point_block)
+                if tool_msg_cache_control is not None:
+                    cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                        {"cache_control": tool_msg_cache_control},
+                        block_type="content_block",
+                        model=model,
+                    )
+                    if cache_point_block is not None:
+                        tool_content.append(cache_point_block)
 
                 msg_i += 1
             # Deduplicate toolResult blocks with the same toolUseId
@@ -4509,6 +4527,7 @@ class BedrockConverseMessagesProcessor:
                         _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
                             message_block=cast(OpenAIMessageContentListBlock, element),
                             block_type="content_block",
+                            model=model,
                         )
                         if _cache_point_block is not None:
                             assistants_parts.append(_cache_point_block)
@@ -4520,14 +4539,14 @@ class BedrockConverseMessagesProcessor:
                     # If content is empty/whitespace, skip it (don't add a placeholder)
                     # Add cache point block for assistant string content
                     _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
-                        assistant_message_block, block_type="content_block"
+                        assistant_message_block, block_type="content_block", model=model
                     )
                     if _cache_point_block is not None:
                         assistant_content.append(_cache_point_block)
 
                 _tool_calls = assistant_message_block.get("tool_calls", [])
                 if _tool_calls:
-                    assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls))
+                    assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model))
 
                 msg_i += 1
 
@@ -4745,6 +4764,7 @@ def _bedrock_converse_messages_pt(
                         _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
                             message_block=cast(OpenAIMessageContentListBlock, element),
                             block_type="content_block",
+                            model=model,
                         )
                         if _cache_point_block is not None:
                             _parts.append(_cache_point_block)
@@ -4752,7 +4772,7 @@ def _bedrock_converse_messages_pt(
             elif message_block["content"] and isinstance(message_block["content"], str):
                 _part = BedrockContentBlock(text=messages[msg_i]["content"])
                 _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
-                    message_block, block_type="content_block"
+                    message_block, block_type="content_block", model=model
                 )
                 user_content.append(_part)
                 if _cache_point_block is not None:
@@ -4786,22 +4806,27 @@ def _bedrock_converse_messages_pt(
             tool_content.append(tool_call_result)
 
             # Check if we need to add a separate cachePoint block
-            has_cache_control = False
+            tool_msg_cache_control = None
 
             # Check for message-level cache_control
             if current_message.get("cache_control", None) is not None:
-                has_cache_control = True
+                tool_msg_cache_control = current_message["cache_control"]
             # Check for content-level cache_control in list content
             elif isinstance(current_message.get("content"), list):
                 for content_element in current_message["content"]:
                     if isinstance(content_element, dict) and content_element.get("cache_control", None) is not None:
-                        has_cache_control = True
+                        tool_msg_cache_control = content_element["cache_control"]
                         break
 
             # Add a separate cachePoint block if cache_control is present
-            if has_cache_control:
-                cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
-                tool_content.append(cache_point_block)
+            if tool_msg_cache_control is not None:
+                cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
+                    {"cache_control": tool_msg_cache_control},
+                    block_type="content_block",
+                    model=model,
+                )
+                if cache_point_block is not None:
+                    tool_content.append(cache_point_block)
 
             msg_i += 1
         # Deduplicate toolResult blocks with the same toolUseId
@@ -4882,6 +4907,7 @@ def _bedrock_converse_messages_pt(
                         _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
                             message_block=cast(OpenAIMessageContentListBlock, element),
                             block_type="content_block",
+                            model=model,
                         )
                         if _cache_point_block is not None:
                             assistants_parts.append(_cache_point_block)
@@ -4892,13 +4918,13 @@ def _bedrock_converse_messages_pt(
                     assistant_content.append(BedrockContentBlock(text=_assistant_content))
                 # Add cache point block for assistant string content
                 _cache_point_block = litellm.AmazonConverseConfig()._get_cache_point_block(
-                    assistant_message_block, block_type="content_block"
+                    assistant_message_block, block_type="content_block", model=model
                 )
                 if _cache_point_block is not None:
                     assistant_content.append(_cache_point_block)
             _tool_calls = assistant_message_block.get("tool_calls", [])
             if _tool_calls:
-                assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls))
+                assistant_content.extend(_convert_to_bedrock_tool_call_invoke(_tool_calls, model=model))
 
             msg_i += 1
 

litellm/litellm_core_utils/realtime_streaming.py

@@ -1,5 +1,4 @@
 import asyncio
-import concurrent.futures
 import json
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union, cast
 
@@ -25,9 +24,6 @@ if TYPE_CHECKING:
 else:
     CLIENT_CONNECTION_CLASS = Any
 
-# Create a thread pool with a maximum of 10 threads
-executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
-
 
 class RealtimeEventNormalizer(Protocol):
     def should_drop(self, event: object) -> bool: ...
@@ -315,13 +311,12 @@ class RealTimeStreaming:
             if self.session_tools or self.tool_calls:
                 self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
                 self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
-            ## ASYNC LOGGING
             # Route through the bounded logging worker (per-coroutine timeout +
             # concurrency cap) instead of a bare create_task, so a slow callback
             # can't leave suspended tasks pinning each call's response in memory.
-            GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(self.logging_obj.async_success_handler(self.messages))
-            ## SYNC LOGGING
-            executor.submit(self.logging_obj.success_handler(self.messages))
+            GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+                self.logging_obj.dispatch_success_handlers(self.messages, prefer_async_handlers=True)
+            )
 
     async def _send_to_backend(self, message: str) -> bool:
         """Send a message to the backend WebSocket.

litellm/llms/bedrock/base_aws_llm.py

@@ -50,6 +50,8 @@ _STS_REGION_FROM_ENDPOINT_PATTERN = re.compile(
     r"(?:^|\.)sts(?:-fips)?\.([a-z0-9-]+)\.(?:amazonaws\.com(?:\.cn)?|vpce\.amazonaws\.com)"
 )
 
+SIGV4_COMPUTED_HEADERS = frozenset({"authorization", "x-amz-date", "x-amz-security-token", "date"})
+
 
 class Boto3CredentialsInfo(BaseModel):
     credentials: Credentials
@@ -1400,11 +1402,13 @@ class BaseAWSLLM:
 
             # Add back all original headers (including forwarded ones) after signature calculation
             for header_name, header_value in headers.items():
-                if header_value is not None:
+                if header_value is not None and header_name.lower() not in SIGV4_COMPUTED_HEADERS:
                     request.headers[header_name] = header_value
 
             if (
-                extra_headers is not None and "Authorization" in extra_headers
+                extra_headers is not None
+                and "Authorization" in extra_headers
+                and not extra_headers["Authorization"].startswith("AWS4-HMAC-SHA256")
             ):  # prevent sigv4 from overwriting the auth header
                 request.headers["Authorization"] = extra_headers["Authorization"]
         prepped = request.prepare()
@@ -1527,9 +1531,15 @@ class BaseAWSLLM:
         # Add back original headers after signing. Only headers in SignedHeaders
         # are integrity-protected; forwarded headers (x-forwarded-*) must remain unsigned.
         for header_name, header_value in headers.items():
-            if header_value is not None:
+            if header_value is not None and header_name.lower() not in SIGV4_COMPUTED_HEADERS:
                 request_headers_dict[header_name] = header_value
-        if headers is not None and "Authorization" in headers:  # prevent sigv4 from overwriting the auth header
-            request_headers_dict["Authorization"] = headers["Authorization"]
+        incoming_authorization = next(
+            (value for name, value in headers.items() if name.lower() == "authorization" and value is not None),
+            None,
+        )
+        if incoming_authorization is not None and not incoming_authorization.startswith(
+            "AWS4-HMAC-SHA256"
+        ):  # prevent sigv4 from overwriting the auth header
+            request_headers_dict["Authorization"] = incoming_authorization
 
         return request_headers_dict, request.body

litellm/llms/bedrock/chat/mantle/transformation.py

@@ -7,13 +7,14 @@ The bedrock-mantle endpoint uses the Anthropic Messages API format but is served
 at a different endpoint (bedrock-mantle.{region}.api.aws) with AWS SigV4 auth.
 """
 
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional
 
 from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
     AmazonAnthropicClaudeConfig,
 )
 from litellm.llms.bedrock.common_utils import build_mantle_messages_url
 from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import ModelResponse
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -91,10 +92,14 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
             litellm_params=litellm_params,
             headers=headers,
         )
-        # The parent strips "model" from the body (Invoke API puts it in URL).
-        # The mantle endpoint (Messages API) requires "model" in the body.
-        request["model"] = model_id
-        return request
+        # The parent strips "model" and "stream" from the body (Invoke API puts
+        # the model in the URL and streams via a dedicated endpoint). The mantle
+        # endpoint (Messages API) requires both in the body.
+        return self._restore_mantle_body_fields(
+            request=request,
+            model_id=model_id,
+            optional_params=optional_params,
+        )
 
     async def async_transform_request(
         self,
@@ -114,5 +119,31 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
             headers=headers,
         )
         await self._async_convert_document_url_sources_to_base64(request)
-        request["model"] = model_id
-        return request
+        return self._restore_mantle_body_fields(
+            request=request,
+            model_id=model_id,
+            optional_params=optional_params,
+        )
+
+    @staticmethod
+    def _restore_mantle_body_fields(request: dict, model_id: str, optional_params: dict) -> dict:
+        stream_fields: dict = {"stream": True} if optional_params.get("stream") is True else {}
+        return {**request, "model": model_id, **stream_fields}
+
+    @property
+    def has_custom_stream_wrapper(self) -> bool:
+        return False
+
+    def get_model_response_iterator(
+        self,
+        streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
+        sync_stream: bool,
+        json_mode: Optional[bool] = False,
+    ) -> Any:
+        from litellm.llms.anthropic.chat.handler import ModelResponseIterator
+
+        return ModelResponseIterator(
+            streaming_response=streaming_response,
+            sync_stream=sync_stream,
+            json_mode=json_mode,
+        )

litellm/llms/bedrock/messages/mantle_transformation.py

@@ -6,8 +6,13 @@ AmazonAnthropicClaudeMessagesConfig. Overrides only the URL and model-prefix
 stripping that are specific to the bedrock-mantle endpoint.
 """
 
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple
 
+import httpx
+
+from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+    AnthropicMessagesConfig,
+)
 from litellm.llms.bedrock.common_utils import build_mantle_messages_url
 from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
     AmazonAnthropicClaudeMessagesConfig,
@@ -89,8 +94,26 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
             headers=headers,
         )
 
-        # Parent (AmazonAnthropicClaudeMessagesConfig) removes "model" from the
-        # body (Bedrock Invoke puts model in the URL). The mantle endpoint
-        # (Messages API) requires "model" in the request body.
-        request["model"] = model_id
-        return request
+        # Parent (AmazonAnthropicClaudeMessagesConfig) removes "model" and
+        # "stream" from the body (Bedrock Invoke puts the model in the URL and
+        # streams via a dedicated endpoint). The mantle endpoint (Messages API)
+        # requires both in the request body.
+        stream_fields: dict[str, bool] = (
+            {"stream": True} if anthropic_messages_optional_request_params.get("stream") is True else {}
+        )
+        return {**request, "model": model_id, **stream_fields}
+
+    def get_async_streaming_response_iterator(
+        self,
+        model: str,
+        httpx_response: httpx.Response,
+        request_body: dict,
+        litellm_logging_obj: LiteLLMLoggingObj,
+    ) -> AsyncIterator:
+        return AnthropicMessagesConfig.get_async_streaming_response_iterator(
+            self,
+            model=model,
+            httpx_response=httpx_response,
+            request_body=request_body,
+            litellm_logging_obj=litellm_logging_obj,
+        )

litellm/llms/vertex_ai/vertex_llm_base.py

@@ -9,6 +9,7 @@ import json
 import os
 import threading
 from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple
+from urllib.parse import urlparse
 
 import litellm
 from litellm._logging import verbose_logger
@@ -315,6 +316,9 @@ class VertexBase:
             api_base=api_base,
         )
 
+        if partner == VertexPartnerProvider.llama:
+            return default_api_base
+
         if len(default_api_base.split(":")) > 1:
             endpoint = default_api_base.split(":")[-1]
         else:
@@ -615,7 +619,8 @@ class VertexBase:
 
         Handles custom api_base for:
         1. Gemini (Google AI Studio) - constructs /models/{model}:{endpoint}
-        2. Vertex AI with standard proxies - constructs {api_base}:{endpoint}
+        2. Vertex AI with standard proxies - constructs {api_base}:{endpoint};
+           if api_base has no path (bare host), grafts the default vertex URL path onto it
         3. Vertex AI with PSC endpoints - constructs full path structure
            {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint}
            (only when use_psc_endpoint_format=True)
@@ -660,8 +665,9 @@ class VertexBase:
                         model_for_url,
                         endpoint,
                     )
+                elif urlparse(api_base).path in ("", "/"):
+                    url = api_base.rstrip("/") + urlparse(url).path
                 else:
-                    # Fallback to simple format if we don't have all parameters
                     url = "{}:{}".format(api_base, endpoint)
             if stream is True:
                 url = url + "?alt=sse"

litellm/main.py

@@ -1082,6 +1082,54 @@ def _build_custom_pricing_entry(
     return entry
 
 
+def _get_router_deployment_id(kwargs: dict) -> Optional[str]:
+    for metadata_key in ("litellm_metadata", "metadata"):
+        metadata = kwargs.get(metadata_key) or {}
+        if not isinstance(metadata, dict):
+            continue
+        deployment_model_info = metadata.get("model_info") or {}
+        if not isinstance(deployment_model_info, dict):
+            continue
+        deployment_id = deployment_model_info.get("id")
+        if deployment_id is not None:
+            return str(deployment_id)
+    return None
+
+
+def _register_custom_pricing_for_request(
+    model: str,
+    custom_llm_provider: str,
+    kwargs: dict,
+    model_info: Optional[dict],
+) -> None:
+    """Register per-request custom pricing in litellm.model_cost.
+
+    Router-originated requests (identified by the deployment id the router puts
+    in metadata) get their full pricing registered under that unique id only;
+    the shared ``{provider}/{model}`` key receives the entry with pricing fields
+    stripped, mirroring Router._create_deployment. This keeps one deployment's
+    pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
+    pricing used by sibling deployments of the same backend model. Direct SDK
+    calls keep the legacy behavior of registering the shared key with pricing.
+    """
+    entry = _build_custom_pricing_entry(
+        custom_llm_provider=custom_llm_provider,
+        kwargs=kwargs,
+        model_info=model_info,
+    )
+    shared_key = f"{custom_llm_provider}/{model}"
+    deployment_id = _get_router_deployment_id(kwargs)
+    if deployment_id is None:
+        litellm.register_model({shared_key: entry})
+        return
+    litellm.register_model(
+        {
+            deployment_id: entry,
+            shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
+        }
+    )
+
+
 def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
     _azure_detection_model = ctx._azure_detection_model
     acompletion = ctx.acompletion
@@ -5108,14 +5156,11 @@ def completion(  # type: ignore
         if (
             input_cost_per_token is not None and output_cost_per_token is not None
         ) or input_cost_per_second is not None:
-            litellm.register_model(
-                {
-                    f"{custom_llm_provider}/{model}": _build_custom_pricing_entry(
-                        custom_llm_provider=custom_llm_provider,
-                        kwargs=kwargs,
-                        model_info=model_info,
-                    )
-                }
+            _register_custom_pricing_for_request(
+                model=model,
+                custom_llm_provider=custom_llm_provider,
+                kwargs=kwargs,
+                model_info=model_info,
             )
         ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
         custom_prompt_dict = {}  # type: ignore
@@ -5959,14 +6004,11 @@ def embedding(
 
     ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
     if (input_cost_per_token is not None and output_cost_per_token is not None) or input_cost_per_second is not None:
-        litellm.register_model(
-            {
-                f"{custom_llm_provider}/{model}": _build_custom_pricing_entry(
-                    custom_llm_provider=custom_llm_provider,
-                    kwargs=kwargs,
-                    model_info=kwargs.get("model_info"),
-                )
-            }
+        _register_custom_pricing_for_request(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            kwargs=kwargs,
+            model_info=kwargs.get("model_info"),
         )
 
     litellm_params_dict = get_litellm_params(**kwargs)

litellm/model_prices_and_context_window_backup.json

@@ -23503,6 +23503,76 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "gpt-realtime-2.1": {
+        "cache_creation_input_audio_token_cost": 4e-07,
+        "cache_read_input_audio_token_cost": 4e-07,
+        "cache_read_input_token_cost": 4e-07,
+        "input_cost_per_audio_token": 3.2e-05,
+        "input_cost_per_image": 5e-06,
+        "input_cost_per_token": 4e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "max_tokens": 32000,
+        "mode": "chat",
+        "output_cost_per_audio_token": 6.4e-05,
+        "output_cost_per_token": 2.4e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/realtime"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "gpt-realtime-2.1-mini": {
+        "cache_creation_input_audio_token_cost": 3e-07,
+        "cache_read_input_audio_token_cost": 3e-07,
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_image": 8e-07,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/realtime"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
     "gpt-realtime-mini": {
         "cache_creation_input_audio_token_cost": 3e-07,
         "cache_read_input_audio_token_cost": 3e-07,
@@ -38031,6 +38101,48 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "xai/grok-4.5": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 1e-06,
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_200k_tokens": 4e-06,
+        "litellm_provider": "xai",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 1.2e-05,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "xai/grok-4.5-latest": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 1e-06,
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_200k_tokens": 4e-06,
+        "litellm_provider": "xai",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 1.2e-05,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "xai/grok-beta": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "xai",

litellm/models/mcp_server.py

@@ -83,6 +83,15 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
     token_url: Optional[str] = None
     registration_url: Optional[str] = None
     oauth2_flow: Optional[Literal["client_credentials", "authorization_code"]] = None
+    # Token Exchange (OBO) fields — RFC 8693. ``audience`` is named for the RFC's
+    # request parameter (token-exchange only); RFC 8707 resource indicators are a
+    # separate concept named ``resource`` in the v2 egress types. A null
+    # ``subject_token_type`` means DEFAULT_SUBJECT_TOKEN_TYPE (litellm.types.mcp),
+    # applied at the egress build sites.
+    token_exchange_endpoint: Optional[str] = None
+    audience: Optional[str] = None
+    subject_token_type: Optional[str] = None
+    token_exchange_profile: Optional[str] = None
     allow_all_keys: bool = False
     available_on_public_internet: bool = True
     delegate_auth_to_upstream: bool = False

litellm/proxy/_experimental/mcp_server/auth/token_exchange.py

@@ -28,6 +28,7 @@ from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import (
     build_token_endpoint_client_auth,
 )
 from litellm.types.llms.custom_http import httpxSpecialProvider
+from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE
 
 if TYPE_CHECKING:
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
@@ -35,8 +36,6 @@ if TYPE_CHECKING:
 # RFC 8693 grant type constant
 TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
 
-DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token"
-
 
 class TokenExchangeHandler:
     """Handles OAuth 2.0 Token Exchange (RFC 8693) for MCP servers.

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -8,6 +8,7 @@ from starlette.types import Scope
 
 from litellm._logging import verbose_logger
 from litellm.proxy._types import (
+    UI_TEAM_ID,
     LiteLLM_TeamTable,
     ProxyException,
     SpecialHeaders,
@@ -219,6 +220,12 @@ class MCPRequestHandler:
             # when EVERY target is auth_type=oauth2 with delegate_auth_to_upstream
             # set; fails closed otherwise.
             validated_user_api_key_auth = UserAPIKeyAuth()
+        elif MCPRequestHandler._target_servers_are_true_passthrough(
+            path=request_route,
+            mcp_servers=mcp_servers,
+            client_ip=IPAddressUtils.get_mcp_client_ip(request),
+        ):
+            validated_user_api_key_auth = UserAPIKeyAuth()
         elif oauth2_headers:
             # Authorization on a non-delegated server: the bearer must be a real
             # LiteLLM credential, so a failed validation is a genuine 401/403 and
@@ -357,6 +364,7 @@ class MCPRequestHandler:
         # Inline imports avoid a circular dependency: mcp_server_manager imports
         # from this module.
         from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+            MCPServerManager,
             global_mcp_server_manager,
         )
         from litellm.types.mcp import MCPAuth
@@ -382,7 +390,45 @@ class MCPRequestHandler:
             # fetches the upstream token automatically using stored credentials,
             # so allowing anonymous bypass would let any external caller invoke
             # tools authenticated as LiteLLM's service account.
-            if server.has_client_credentials:
+            #
+            # Resolve the flow rather than reading has_client_credentials directly:
+            # this is a security gate, and a legacy row whose oauth2_flow was never
+            # stamped still carries the M2M credential shape (client_id/secret +
+            # token_url, no authorization_url). Treating an unstamped-but-M2M-shaped
+            # row as non-M2M here would reopen the anonymous bypass the explicit
+            # column no longer closes on its own. Shares the one resolution helper
+            # with the egress backstop and the anonymous-delegate allowlist; all fail
+            # closed on the ambiguous shape and are removed together once no null rows
+            # remain. A pure-PKCE delegate server (no stored credentials) resolves to a
+            # non-M2M flow and keeps its bypass.
+            if MCPServerManager.effective_oauth2_flow(server) == "client_credentials":
+                return False
+        return True
+
+    @staticmethod
+    def _target_servers_are_true_passthrough(
+        path: str, mcp_servers: Optional[list[str]], client_ip: Optional[str]
+    ) -> bool:
+        """
+        True only when EVERY MCP server the request targets is ``auth_type == true_passthrough``.
+        Fails closed when any target does not opt in or cannot be resolved.
+
+        Used by :meth:`process_mcp_request` to skip LiteLLM admission auth entirely: the gateway is a
+        transparent proxy and the caller's ``Authorization`` is an upstream token, never a LiteLLM key.
+        Mirrors :meth:`_target_servers_delegate_auth_to_upstream`; a mixed-target request keeps normal auth.
+        """
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+            global_mcp_server_manager,
+        )
+        from litellm.types.mcp import MCPAuth
+
+        target_names = MCPRequestHandler._resolve_target_server_names(path=path, mcp_servers_header=mcp_servers)
+        if not target_names:
+            return False
+
+        for name in target_names:
+            server = global_mcp_server_manager.get_mcp_server_by_name(name, client_ip=client_ip)
+            if server is None or server.auth_type != MCPAuth.true_passthrough:
                 return False
         return True
 
@@ -545,10 +591,19 @@ class MCPRequestHandler:
 
         ASGI headers are in format: List[List[bytes, bytes]]
         We need to convert them to the format Headers expects.
+
+        Collapsing the ASGI list into a dict keeps the last value for a duplicated
+        header name, so a request carrying more than one ``Authorization`` is
+        rejected first: for the client-forwarded token modes the gateway relays the
+        caller's ``Authorization`` upstream, so a duplicate would make which token is
+        forwarded ambiguous (and diverge from what admission inspected). Multiple
+        ``Authorization`` headers is malformed for bearer auth anyway (RFC 9110: not
+        a comma-combinable field), so fail closed with a 400.
         """
+        raw_headers = scope.get("headers", [])
+        MCPRequestHandler._reject_duplicate_authorization(raw_headers)
         try:
             # ASGI headers are list of [name: bytes, value: bytes] pairs
-            raw_headers = scope.get("headers", [])
             # Convert bytes to strings and create dict for Headers constructor
             headers_dict = {name.decode("latin-1"): value.decode("latin-1") for name, value in raw_headers}
             return Headers(headers_dict)
@@ -557,6 +612,26 @@ class MCPRequestHandler:
             # Return empty Headers object with empty dict
             return Headers({})
 
+    @staticmethod
+    def _reject_duplicate_authorization(raw_headers: object) -> None:
+        """Raise 400 when the raw ASGI headers carry more than one ``Authorization`` header."""
+        if not isinstance(raw_headers, (list, tuple)):
+            return
+        count = 0
+        for entry in raw_headers:
+            if not isinstance(entry, (list, tuple)) or len(entry) < 1:
+                continue
+            name = entry[0]
+            if isinstance(name, (bytes, bytearray)) and bytes(name).lower() == b"authorization":
+                count += 1
+            elif isinstance(name, str) and name.lower() == "authorization":
+                count += 1
+        if count > 1:
+            raise HTTPException(
+                status_code=400,
+                detail="Multiple Authorization headers are not allowed",
+            )
+
     @staticmethod
     async def get_allowed_mcp_servers(
         user_api_key_auth: Optional[UserAPIKeyAuth] = None,
@@ -726,6 +801,9 @@ class MCPRequestHandler:
         if not user_api_key_auth or not user_api_key_auth.team_id or not prisma_client:
             return None
 
+        if user_api_key_auth.team_id == UI_TEAM_ID:
+            return None
+
         # Get the team object (which has object_permission already loaded)
         team_obj: Optional[LiteLLM_TeamTable] = await get_team_object(
             team_id=user_api_key_auth.team_id,
@@ -1021,6 +1099,9 @@ class MCPRequestHandler:
             if user_api_key_auth is None or not user_api_key_auth.team_id or prisma_client is None:
                 return []
 
+            if user_api_key_auth.team_id == UI_TEAM_ID:
+                return []
+
             team_obj: Optional[LiteLLM_TeamTable] = await get_team_object(
                 team_id=user_api_key_auth.team_id,
                 prisma_client=prisma_client,
@@ -1503,6 +1584,9 @@ class MCPRequestHandler:
             verbose_logger.debug("prisma_client is None")
             return []
 
+        if user_api_key_auth.team_id == UI_TEAM_ID:
+            return []
+
         try:
             team_obj: Optional[LiteLLM_TeamTable] = await get_team_object(
                 team_id=user_api_key_auth.team_id,

litellm/proxy/_experimental/mcp_server/db.py

@@ -46,6 +46,35 @@ from litellm.types.mcp import MCPCredentials
 if TYPE_CHECKING:
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
+_AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset(
+    {
+        "authorization_url",
+        "token_url",
+        "registration_url",
+        "oauth2_flow",
+        "token_exchange_endpoint",
+        "audience",
+        "subject_token_type",
+        "token_exchange_profile",
+    }
+)
+
+# Token-exchange settings with dedicated columns that also exist on
+# ``MCPCredentials`` as a legacy shape (rows and REST callers that predate the
+# columns). Every write lifts blob values into the columns and strips them from
+# the stored blob, so the read-time ``column or blob`` fallback only serves rows
+# the current code has never written — a cleared column can then never be
+# silently resurrected by a stale blob copy. These keys are stored plaintext
+# (endpoints/identifiers, not secrets), so values lift as-is.
+_TOKEN_EXCHANGE_COLUMN_FIELDS: frozenset = frozenset(
+    {
+        "token_exchange_endpoint",
+        "audience",
+        "subject_token_type",
+        "token_exchange_profile",
+    }
+)
+
 
 def _is_global_env_var_scope(scope: Any) -> bool:
     """``scope="user"`` entries are placeholders the user fills in; everything
@@ -241,6 +270,14 @@ def _prepare_mcp_server_data(
     # Handle credentials serialization
     credentials = data_dict.get("credentials")
     if credentials is not None:
+        # Lift legacy blob-shaped token-exchange settings into their dedicated
+        # columns (an explicit top-level value wins, including an explicit
+        # null) and strip them from the blob so it never seeds the read-time
+        # fallback for rows written by current code.
+        for te_field in _TOKEN_EXCHANGE_COLUMN_FIELDS:
+            blob_value = credentials.pop(te_field, None)
+            if blob_value is not None and te_field not in data_dict:
+                data_dict[te_field] = blob_value
         data_dict["credentials"] = encrypt_credentials(credentials=credentials, encryption_key=_get_salt_key())
         data_dict["credentials"] = safe_dumps(data_dict["credentials"])
 
@@ -603,19 +640,41 @@ async def update_mcp_server(
     # Pre-fetch existing record once if we need it for auth_type or credential logic
     existing = None
     has_credentials = "credentials" in data_dict and data_dict["credentials"] is not None
-    if data.auth_type or has_credentials:
+    # An explicit token-exchange column write (set or clear) also migrates the
+    # legacy blob copies below, so the existing row is needed for those updates.
+    explicit_te_write = bool(_TOKEN_EXCHANGE_COLUMN_FIELDS & data_dict.keys())
+    if data.auth_type or has_credentials or explicit_te_write:
         existing = await MCPServerRepository(prisma_client).table.find_unique(where={"server_id": data.server_id})
 
+    auth_type_changed = bool(
+        data.auth_type and existing and existing.auth_type is not None and existing.auth_type != data.auth_type
+    )
+
     # Clear stale credentials when auth_type changes but no new credentials provided
-    if (
-        data.auth_type
-        and "credentials" not in data_dict
-        and existing
-        and existing.auth_type is not None
-        and existing.auth_type != data.auth_type
-    ):
+    if auth_type_changed and "credentials" not in data_dict:
         data_dict["credentials"] = None
 
+    if auth_type_changed:
+        data_dict.update({field: None for field in _AUTH_FLOW_SCOPED_FIELDS if field not in data_dict})
+
+    # An explicit column write that does not touch credentials must still migrate
+    # the row's legacy blob copies: lift values for columns the caller left
+    # untouched, strip every copy from the blob. Without this, clearing a column
+    # (e.g. to re-enable RFC 9728/8414 discovery) would leave the blob copy in
+    # place, and the next credentials update's migrate-on-write would silently
+    # repopulate the column the admin just cleared. (When credentials ARE in the
+    # update, the merge below performs the same migration.)
+    if explicit_te_write and "credentials" not in data_dict and existing is not None and existing.credentials:
+        existing_creds = (
+            json.loads(existing.credentials) if isinstance(existing.credentials, str) else dict(existing.credentials)
+        )
+        if _TOKEN_EXCHANGE_COLUMN_FIELDS & existing_creds.keys():
+            for te_field in _TOKEN_EXCHANGE_COLUMN_FIELDS:
+                legacy_value = existing_creds.pop(te_field, None)
+                if legacy_value is not None and te_field not in data_dict and getattr(existing, te_field, None) is None:
+                    data_dict[te_field] = legacy_value
+            data_dict["credentials"] = safe_dumps(existing_creds)
+
     # Merge credentials: preserve existing fields not present in the update.
     # Without this, a partial credential update (e.g. changing only region)
     # would wipe encrypted secrets that the UI cannot display back.
@@ -638,6 +697,19 @@ async def update_mcp_server(
                 )
                 # New values override existing; existing keys not in update are preserved
                 merged = {**existing_creds, **new_creds}
+                # Migrate-on-write for legacy rows: token-exchange settings the
+                # old blob shape carried move to their dedicated columns (unless
+                # the caller set the column this update, or the row already has
+                # one) and are never re-persisted in the blob. Stored plaintext,
+                # so the merged value lifts as-is.
+                for te_field in _TOKEN_EXCHANGE_COLUMN_FIELDS:
+                    legacy_value = merged.pop(te_field, None)
+                    if (
+                        legacy_value is not None
+                        and te_field not in data_dict
+                        and getattr(existing, te_field, None) is None
+                    ):
+                        data_dict[te_field] = legacy_value
                 data_dict["credentials"] = safe_dumps(merged)
 
     # Add audit fields

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -448,6 +448,12 @@ async def _store_per_user_token_server_side(
         )
         return  # Don't warm Redis if DB write failed
 
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415
+        global_mcp_server_manager,
+    )
+
+    await global_mcp_server_manager.invalidate_user_oauth_token_cache(user_id, server.server_id)
+
     # Warm the Redis cache so the first subsequent MCP call is a cache hit
     ttl = _compute_per_user_token_ttl(server, expires_in)
     await mcp_per_user_token_cache.set(
@@ -575,8 +581,12 @@ async def exchange_token_with_server(
     if mcp_server.token_url is None:
         raise HTTPException(status_code=400, detail="MCP server token url is not set")
 
+    # The id and secret must come from the same source. When the server-side client_id wins,
+    # falling back to the caller's secret pairs the persisted client with a foreign secret; the
+    # register short-circuit hands clients a placeholder secret ("dummy"), so a re-auth against a
+    # persisted public PKCE client (no stored secret) would send that placeholder and the IdP 401s.
     resolved_client_id = mcp_server.client_id if mcp_server.client_id else client_id
-    resolved_client_secret = mcp_server.client_secret if mcp_server.client_secret else client_secret
+    resolved_client_secret = mcp_server.client_secret if mcp_server.client_id else client_secret
     try:
         client_auth = build_token_endpoint_client_auth(
             auth_method=mcp_server.token_endpoint_auth_method,
@@ -1292,11 +1302,15 @@ async def _build_oauth_protected_resource_response(
     """
     Build OAuth protected resource response with the appropriate URL pattern.
 
-    For pass-through MCP servers (``MCPServer.is_oauth_passthrough``), the
-    gateway proxies the upstream's own ``oauth-protected-resource`` metadata
-    so that standards-compliant MCP clients discover the **upstream** IdP
-    instead of the gateway. The ``resource`` field is rewritten to the
-    gateway's own URL so clients present the bearer token back to the gateway.
+    For pass-through MCP servers, the gateway proxies the upstream's own
+    ``oauth-protected-resource`` metadata so standards-compliant MCP clients
+    discover the **upstream** IdP instead of the gateway. For ``true_passthrough``
+    and ``oauth_delegate`` the metadata is returned verbatim (``resource`` stays
+    the upstream): the caller's token is forwarded to and validated by the
+    upstream, so its audience must be the upstream — rewriting it to the gateway
+    would make a strict IdP (e.g. Entra) refuse to mint it or the upstream reject
+    it. Only the legacy ``is_oauth_passthrough`` opt-in rewrites ``resource`` to
+    the gateway's own URL so clients present the bearer token back to the gateway.
 
     Args:
         request: FastAPI Request object
@@ -1337,7 +1351,9 @@ async def _build_oauth_protected_resource_response(
 
     # Pass-through branch: proxy the upstream's own metadata so discovery
     # directs the client at the real IdP (Okta, Keycloak, …) instead of us.
-    if mcp_server is not None and mcp_server.is_oauth_passthrough:
+    if mcp_server is not None and (
+        mcp_server.is_oauth_passthrough or mcp_server.is_oauth_delegate or mcp_server.is_true_passthrough
+    ):
         try:
             upstream_metadata = await fetch_upstream_oauth_protected_resource(mcp_server)
         except Exception as exc:
@@ -1353,8 +1369,9 @@ async def _build_oauth_protected_resource_response(
             )
 
         if upstream_metadata is not None:
-            response = {**upstream_metadata, "resource": resource_url}
-            return response
+            if mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+                return upstream_metadata
+            return {**upstream_metadata, "resource": resource_url}
 
         # Upstream responded but with non-200 or non-dict payload. For
         # pass-through servers the gateway is NOT the authorization server,

litellm/proxy/_experimental/mcp_server/exceptions.py

@@ -73,3 +73,18 @@ class MCPUpstreamAuthError(Exception):
             detail=detail,
             headers={"www-authenticate": challenge} if challenge else None,
         )
+
+
+class MCPToolResultError(Exception):
+    """An MCP tool call completed with ``isError=True`` in its result.
+
+    Never raised on the wire path: streamable HTTP MCP correctly returns tool
+    failures as HTTP 200 with ``result.isError: true`` per the MCP spec. This
+    exception only drives the standard failure logging (``status="failure"``
+    payload, OTel ERROR span) for such results.
+
+    Lives here rather than ``utils.py`` deliberately: tests reload ``utils``
+    to re-read its env-derived constants, and a reload would fork this class
+    into two identities, breaking ``isinstance`` checks against instances
+    created before the reload.
+    """

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -70,6 +70,9 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.adapter import
     to_server_spec,
     to_subject,
 )
+from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
+    InvalidatableOAuthTokenStore,
+)
 from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import (
     LazyPerUserOAuthTokenStore,
 )
@@ -78,6 +81,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.token_exchange_
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     AuthorizationCodeConfig,
+    PassthroughConfig,
     ServerSpec,
     TokenExchangeConfig,
 )
@@ -115,7 +119,7 @@ from litellm.proxy.common_utils.user_api_key_cache import get_management_object_
 from litellm.proxy.utils import ProxyLogging, get_server_root_path
 from litellm.repositories.table_repositories import MCPServerRepository
 from litellm.types.llms.custom_http import httpxSpecialProvider
-from litellm.types.mcp import MCPAuth, MCPStdioConfig
+from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE, MCPAuth, MCPStdioConfig
 from litellm.types.mcp_server.mcp_server_manager import (
     MCPInfo,
     MCPOAuthMetadata,
@@ -213,6 +217,13 @@ def _should_strip_caller_authorization(
       pass-through cold-start case (RFC 9728) the bearer in
       ``Authorization`` is the upstream OAuth token and must be
       forwarded, so we keep it.
+    - **oauth_delegate servers**: admission always runs and there is no
+      anonymous path, so the caller's separate ``Authorization`` is
+      forwarded only when a distinct ``x-litellm-api-key`` carried
+      admission. Without that header the ``Authorization`` *was* the
+      admission credential — a virtual key, an IdP JWT, or an SSO / OIDC /
+      session token whose ``api_key`` is ``None`` — and must never reach
+      the upstream, so it is stripped regardless of the ``api_key`` value.
     """
     if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
         # OBO: the inbound Authorization is the subject token. It is exchanged at the IdP and only the
@@ -226,11 +237,13 @@ def _should_strip_caller_authorization(
         # upstream — it would override another user's stored credential. Delegate and
         # pass-through return None from to_server_spec and keep forwarding the bearer.
         return True
-    if not mcp_server.is_oauth_passthrough:
+    if not (mcp_server.is_oauth_passthrough or mcp_server.is_oauth_delegate):
         return False
 
     normalized_raw_headers = {str(k).lower(): v for k, v in (raw_headers or {}).items() if isinstance(k, str)}
     has_explicit_litellm_admission_header = normalized_raw_headers.get("x-litellm-api-key") is not None
+    if mcp_server.is_oauth_delegate:
+        return not has_explicit_litellm_admission_header
     admission_consumed_authorization_as_litellm_key = (
         user_api_key_auth is not None
         and bool(getattr(user_api_key_auth, "api_key", None))
@@ -253,6 +266,159 @@ def _without_authorization(
     return filtered or None
 
 
+def _format_byok_openapi_auth_header(mcp_server: MCPServer, mcp_auth_header: str) -> str:
+    """Format a raw BYOK credential for OpenAPI tool ``Authorization`` injection."""
+    if mcp_server.auth_type == MCPAuth.api_key:
+        return f"ApiKey {mcp_auth_header}"
+    if mcp_server.auth_type == MCPAuth.basic:
+        return f"Basic {mcp_auth_header}"
+    return f"Bearer {mcp_auth_header}"
+
+
+def _openapi_forwarded_extra_headers(
+    mcp_server: MCPServer,
+    raw_headers: Optional[dict[str, str]],
+    user_api_key_auth: Optional[UserAPIKeyAuth],
+) -> Optional[dict[str, str]]:
+    if not mcp_server.extra_headers or not raw_headers:
+        return None
+    normalized_raw = {str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str)}
+    skip_caller_authorization = _should_strip_caller_authorization(
+        mcp_server=mcp_server,
+        raw_headers=raw_headers,
+        user_api_key_auth=user_api_key_auth,
+    )
+    forwarded: dict[str, str] = {}
+    for header_name in mcp_server.extra_headers:
+        if not isinstance(header_name, str):
+            continue
+        if skip_caller_authorization and header_name.lower() == "authorization":
+            continue
+        value = normalized_raw.get(header_name.lower())
+        if value is not None:
+            forwarded[header_name] = value
+    return forwarded or None
+
+
+async def _resolve_byok_mcp_auth_header(
+    mcp_server: MCPServer,
+    user_api_key_auth: Optional[UserAPIKeyAuth],
+    mcp_auth_header: Optional[str],
+) -> Optional[str]:
+    """Resolve BYOK credential for tool calls that bypass ``execute_mcp_tool``."""
+    if not mcp_server.is_byok:
+        return mcp_auth_header
+
+    from litellm.proxy._experimental.mcp_server.server import (
+        _check_byok_credential,
+        _get_byok_credential,
+    )
+
+    if not mcp_auth_header:
+        byok_cred = await _get_byok_credential(mcp_server, user_api_key_auth)
+        if byok_cred is None:
+            raise HTTPException(
+                status_code=401,
+                detail={
+                    "error": "byok_auth_required",
+                    "server_id": mcp_server.server_id,
+                    "server_name": mcp_server.server_name or mcp_server.name,
+                    "message": (
+                        "No stored credential found for this BYOK server. "
+                        "Complete the OAuth authorization flow to provide your API key."
+                    ),
+                },
+                headers={"WWW-Authenticate": 'Bearer resource_metadata="/.well-known/oauth-protected-resource"'},
+            )
+        return byok_cred
+
+    await _check_byok_credential(mcp_server, user_api_key_auth)
+    return mcp_auth_header
+
+
+def _client_forwarded_authorization_headers(
+    mcp_server: MCPServer,
+    oauth2_headers: Optional[dict[str, str]],
+    raw_headers: Optional[dict[str, str]],
+    user_api_key_auth: Optional[UserAPIKeyAuth],
+) -> Optional[dict[str, str]]:
+    """Egress headers for the client-forwarded-token modes (``true_passthrough`` / ``oauth_delegate``).
+
+    Forwards the caller's ``Authorization`` to the upstream, stripped when
+    ``_should_strip_caller_authorization`` says it was consumed as the LiteLLM admission key. Shared by
+    ``_call_regular_mcp_tool`` and ``server.py``'s ``_prepare_mcp_server_headers`` so the two egress
+    paths cannot drift, mirroring the ``_should_strip_caller_authorization`` split.
+    """
+    extra_headers = oauth2_headers.copy() if oauth2_headers else None
+    if extra_headers and _should_strip_caller_authorization(
+        mcp_server=mcp_server,
+        raw_headers=raw_headers,
+        user_api_key_auth=user_api_key_auth,
+    ):
+        return _without_authorization(extra_headers)
+    return extra_headers
+
+
+def _take_forwarded_authorization(
+    headers: Optional[dict[str, str]],
+) -> tuple[Optional[str], Optional[dict[str, str]]]:
+    """Pop the ``Authorization`` value out of ``headers`` (case-insensitive), returning it with the
+    remaining headers, so the passthrough resolver arm is the single Authorization source rather than
+    the header also riding in ``extra_headers`` (which the resolved auth would then defer to)."""
+    if not headers:
+        return None, headers
+    value = next((v for k, v in headers.items() if k.lower() == "authorization"), None)
+    return value, _without_authorization(headers)
+
+
+def _passthrough_token_from_mcp_auth_header(
+    mcp_auth_header: Optional[Union[str, dict[str, str]]],
+) -> Optional[str]:
+    """The caller's per-server upstream credential for a passthrough-mode server, or None.
+
+    Sourced from ``x-mcp-{alias}-authorization`` (string or per-header dict form) or the deprecated
+    global ``x-mcp-auth`` fallback. Per-server headers are the multi-server shape: they bind one
+    token to one server, so an aggregate scope with several passthrough-mode servers never replays
+    a single credential across upstreams. The value is forwarded verbatim, so it must be the full
+    header value (e.g. ``Bearer <upstream-token>``)."""
+    if isinstance(mcp_auth_header, str):
+        return mcp_auth_header or None
+    if isinstance(mcp_auth_header, dict):
+        return next((v for k, v in mcp_auth_header.items() if k.lower() == "authorization"), None)
+    return None
+
+
+def _consumes_caller_authorization(server: MCPServer) -> bool:
+    """True when this server's egress forwards the caller's request-wide ``Authorization`` upstream:
+    the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated
+    interactive oauth2. An unstamped oauth2 row (flow column not yet backfilled) reads as a consumer,
+    which errs toward suppression — the fail-safe direction."""
+    if server.is_true_passthrough or server.is_oauth_delegate or server.is_oauth_passthrough:
+        return True
+    return (
+        server.auth_type == MCPAuth.oauth2
+        and getattr(server, "delegate_auth_to_upstream", False) is True
+        and not server.has_client_credentials
+    )
+
+
+def _caller_authorization_fans_out(
+    server: MCPServer,
+    scope_servers: Optional[list[MCPServer]],
+) -> bool:
+    """True when forwarding the caller's request-wide ``Authorization`` to ``server`` inside a
+    listing fan-out would replay one credential against multiple upstreams: another server in the
+    scope also consumes it (RFC 9700 cross-resource replay). ``scope_servers`` is None for
+    explicitly-addressed operations (tool call, get_prompt, read_resource, single-server routes),
+    where the client named the one target and the gateway is not choosing recipients."""
+    if scope_servers is None:
+        return False
+    return any(
+        other is not None and other.server_id != server.server_id and _consumes_caller_authorization(other)
+        for other in scope_servers
+    )
+
+
 def _extract_upstream_auth_failure(
     exc: BaseException,
 ) -> Optional[tuple[int, Optional[str]]]:
@@ -511,6 +677,21 @@ def _create_elicitation_callback():
 class MCPServerManager:
     _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$")
 
+    @staticmethod
+    def _explicit_oauth2_flow(
+        oauth2_flow: Optional[str],
+    ) -> Optional[Literal["client_credentials", "authorization_code"]]:
+        """DB rows persist their flow (write-time stamps plus the startup backfill) and
+        config servers must declare it (validated at load), so both builds read the
+        value verbatim: unknown or null resolves to None, which
+        ``needs_user_oauth_token`` already treats as interactive. Field-shape inference
+        survives only in the request-time security helpers (``effective_oauth2_flow`` /
+        ``resolve_oauth2_flow_for_request``).
+        """
+        if oauth2_flow in ("client_credentials", "authorization_code"):
+            return cast(Literal["client_credentials", "authorization_code"], oauth2_flow)
+        return None
+
     @staticmethod
     def _resolve_oauth2_flow(
         *,
@@ -521,11 +702,18 @@ class MCPServerManager:
         client_id: Optional[str],
         client_secret: Optional[str],
     ) -> Optional[Literal["client_credentials", "authorization_code"]]:
-        """Infer oauth2_flow for legacy records that omit the field.
+        """Infer oauth2_flow from field shape when the value is omitted.
 
-        DB rows created before oauth2_flow support may have OAuth2 client
-        credentials + token_url but a null oauth2_flow. Treat these as M2M,
-        unless authorization_url is present (interactive OAuth).
+        SECURITY-SENSITIVE: this is the shape-inference engine both request-time security
+        helpers delegate to, so it is what decides M2M-vs-interactive for an unstamped row.
+        Always access it through ``effective_oauth2_flow`` (boolean/enum decisions) or
+        ``resolve_oauth2_flow_for_request`` (the egress object backstop), which are the single
+        choke points for request-time resolution; do not call it directly from security sites
+        and do not weaken its M2M-shape branch without accounting for those callers. DB rows
+        are stamped at write time and by the startup backfill, config servers must declare
+        oauth2_flow (validated at load), and both builds read the value verbatim via
+        ``_explicit_oauth2_flow``. Delete this whole request-time layer only once the backstop
+        warning stays silent in production.
         """
         if oauth2_flow in ("client_credentials", "authorization_code"):
             return cast(Literal["client_credentials", "authorization_code"], oauth2_flow)
@@ -540,6 +728,51 @@ class MCPServerManager:
             return "client_credentials"
         return None
 
+    @staticmethod
+    def effective_oauth2_flow(server: "MCPServer") -> Optional[Literal["client_credentials", "authorization_code"]]:
+        """The oauth2_flow a security decision must use for ``server`` this request.
+
+        Column-first, shape-fallback: a stamped row returns its explicit value; an
+        unstamped (null) row whose fields carry the M2M shape resolves to
+        ``client_credentials`` so it is treated as M2M and fails closed. Every
+        security-sensitive reader (anonymous-delegate allowlist and gate, egress flow
+        resolution) goes through this one helper rather than reading the bare
+        ``has_client_credentials`` column, which is unreliable for null rows.
+        """
+        return MCPServerManager._resolve_oauth2_flow(
+            auth_type=server.auth_type,
+            oauth2_flow=server.oauth2_flow,
+            token_url=server.token_url,
+            authorization_url=server.authorization_url,
+            client_id=server.client_id,
+            client_secret=server.client_secret,
+        )
+
+    @staticmethod
+    def resolve_oauth2_flow_for_request(server: "MCPServer") -> "MCPServer":
+        """Return ``server`` with its effective oauth2_flow applied, for egress paths.
+
+        A stamped row is returned unchanged (its effective flow equals the stored value).
+        An unstamped M2M-shape row is returned as a per-request copy carrying
+        ``oauth2_flow=client_credentials`` so downstream ``has_client_credentials`` /
+        ``needs_user_oauth_token`` compute correctly and the stored client credentials are
+        used instead of forwarding the caller's Authorization. Use this at every point that
+        resolves an allowed server id into an ``MCPServer`` for a tool call or listing.
+        """
+        effective = MCPServerManager.effective_oauth2_flow(server)
+        if effective is None or effective == server.oauth2_flow:
+            return server
+        verbose_logger.warning(
+            "MCP server %s has no persisted oauth2_flow but matches the %s shape; using the "
+            "inferred flow for this request. The startup backfill leaves this ambiguous M2M "
+            "shape unstamped on purpose, so it will NOT self-heal: set oauth2_flow explicitly "
+            "in the dashboard or via PUT /v1/mcp/server (client_credentials for M2M, or "
+            "authorization_code after an interactive sign-in).",
+            server.server_id,
+            effective,
+        )
+        return server.model_copy(update={"oauth2_flow": effective})
+
     @staticmethod
     def _obo_needs_endpoint_discovery(
         auth_type: Optional[MCPAuthType],
@@ -552,9 +785,16 @@ class MCPServerManager:
         """
         return auth_type == MCPAuth.oauth2_token_exchange and not (token_exchange_endpoint or token_url)
 
-    def __init__(self, cred_provider: Optional[UpstreamCredentialProvider] = None):
+    def __init__(
+        self,
+        cred_provider: Optional[UpstreamCredentialProvider] = None,
+        per_user_oauth_token_store: Optional[InvalidatableOAuthTokenStore] = None,
+    ):
+        self._per_user_oauth_token_store = per_user_oauth_token_store or LazyPerUserOAuthTokenStore(
+            self.get_mcp_server_by_id
+        )
         self._cred_provider = cred_provider or UpstreamCredentialProvider(
-            oauth_token_store=LazyPerUserOAuthTokenStore(self.get_mcp_server_by_id),
+            oauth_token_store=self._per_user_oauth_token_store,
             token_exchanger=build_token_exchanger(),
         )
         self.registry: dict[str, MCPServer] = {}
@@ -578,8 +818,10 @@ class MCPServerManager:
         # Per-server outbound tool-call concurrency limiters, lazily created from
         # each server's max_concurrent_requests. Keyed by server_id so the cap
         # survives the registry atomic-swap on config reload; a missing key means
-        # the server has no configured limit.
-        self._server_call_semaphores: dict[str, asyncio.Semaphore] = {}
+        # the server has no configured limit. The limit is cached alongside the
+        # semaphore so an edited limit rebuilds it instead of keeping the old cap
+        # until restart.
+        self._server_call_semaphores: dict[str, tuple[int, asyncio.Semaphore]] = {}
         self.tool_name_to_mcp_server_name_mapping: dict[str, str] = {}
         """
         {
@@ -772,6 +1014,20 @@ class MCPServerManager:
                 mcp_oauth_metadata.registration_url if mcp_oauth_metadata else None
             )
 
+            config_oauth2_flow = server_config.get("oauth2_flow", None)
+            if auth_type == MCPAuth.oauth2 and config_oauth2_flow not in (
+                "client_credentials",
+                "authorization_code",
+            ):
+                raise ValueError(
+                    f"Invalid config for MCP server '{server_name or server_id}': auth_type oauth2 "
+                    f"requires an explicit oauth2_flow (got {config_oauth2_flow!r}). Set "
+                    "oauth2_flow: client_credentials for machine-to-machine servers (the proxy mints "
+                    "a shared token at token_url using client_id/client_secret, no user interaction) "
+                    "or oauth2_flow: authorization_code for interactive servers (per-user tokens via "
+                    "browser sign-in, including delegate_auth_to_upstream)."
+                )
+
             new_server = MCPServer(
                 server_id=server_id,
                 name=name_for_prefix,
@@ -785,14 +1041,7 @@ class MCPServerManager:
                 # oauth specific fields
                 client_id=server_config.get("client_id", None),
                 client_secret=server_config.get("client_secret", None),
-                oauth2_flow=self._resolve_oauth2_flow(
-                    auth_type=auth_type,
-                    oauth2_flow=server_config.get("oauth2_flow", None),
-                    token_url=resolved_token_url,
-                    authorization_url=resolved_authorization_url,
-                    client_id=server_config.get("client_id", None),
-                    client_secret=server_config.get("client_secret", None),
-                ),
+                oauth2_flow=self._explicit_oauth2_flow(config_oauth2_flow),
                 scopes=resolved_scopes,
                 authorization_url=resolved_authorization_url,
                 token_url=resolved_token_url,
@@ -828,7 +1077,7 @@ class MCPServerManager:
                 audience=server_config.get("audience", None),
                 subject_token_type=server_config.get(
                     "subject_token_type",
-                    "urn:ietf:params:oauth:token-type:access_token",
+                    DEFAULT_SUBJECT_TOKEN_TYPE,
                 ),
                 token_exchange_profile=server_config.get("token_exchange_profile", "rfc8693"),
                 allow_sampling=bool(server_config.get("allow_sampling", False)),
@@ -1139,7 +1388,8 @@ class MCPServerManager:
             (auth_type == MCPAuth.oauth2 and not mcp_server.authorization_url)
             or self._obo_needs_endpoint_discovery(
                 auth_type,
-                credentials_dict.get("token_exchange_endpoint") if credentials_dict else None,
+                mcp_server.token_exchange_endpoint
+                or (credentials_dict.get("token_exchange_endpoint") if credentials_dict else None),
                 mcp_server.token_url,
             )
         )
@@ -1170,15 +1420,7 @@ class MCPServerManager:
             env_vars=env_vars_list,
             client_id=client_id_value or getattr(mcp_server, "client_id", None),
             client_secret=client_secret_value or getattr(mcp_server, "client_secret", None),
-            oauth2_flow=self._resolve_oauth2_flow(
-                auth_type=auth_type,
-                oauth2_flow=getattr(mcp_server, "oauth2_flow", None),
-                token_url=mcp_server.token_url or getattr(mcp_oauth_metadata, "token_url", None),
-                authorization_url=mcp_server.authorization_url
-                or getattr(mcp_oauth_metadata, "authorization_url", None),
-                client_id=client_id_value or getattr(mcp_server, "client_id", None),
-                client_secret=client_secret_value or getattr(mcp_server, "client_secret", None),
-            ),
+            oauth2_flow=self._explicit_oauth2_flow(getattr(mcp_server, "oauth2_flow", None)),
             scopes=resolved_scopes,
             authorization_url=mcp_server.authorization_url or getattr(mcp_oauth_metadata, "authorization_url", None),
             token_url=mcp_server.token_url or getattr(mcp_oauth_metadata, "token_url", None),
@@ -1213,12 +1455,16 @@ class MCPServerManager:
             aws_role_name=aws_creds.get("aws_role_name"),
             aws_session_name=aws_creds.get("aws_session_name"),
             instructions=mcp_server.instructions,
-            # Token Exchange (OBO) fields — read from credentials JSON blob
-            token_exchange_endpoint=(credentials_dict.get("token_exchange_endpoint") if credentials_dict else None),
-            audience=(credentials_dict.get("audience") if credentials_dict else None),
-            subject_token_type=(credentials_dict.get("subject_token_type") if credentials_dict else None)
-            or "urn:ietf:params:oauth:token-type:access_token",
-            token_exchange_profile=(credentials_dict.get("token_exchange_profile") if credentials_dict else None)
+            # Token exchange (OBO) fields: dedicated columns, with the credentials blob as a
+            # back-compat fallback for servers persisted before the columns existed.
+            token_exchange_endpoint=mcp_server.token_exchange_endpoint
+            or (credentials_dict.get("token_exchange_endpoint") if credentials_dict else None),
+            audience=mcp_server.audience or (credentials_dict.get("audience") if credentials_dict else None),
+            subject_token_type=mcp_server.subject_token_type
+            or (credentials_dict.get("subject_token_type") if credentials_dict else None)
+            or DEFAULT_SUBJECT_TOKEN_TYPE,
+            token_exchange_profile=mcp_server.token_exchange_profile
+            or (credentials_dict.get("token_exchange_profile") if credentials_dict else None)
             or "rfc8693",
             timeout=getattr(mcp_server, "timeout", None),
             max_concurrent_requests=getattr(mcp_server, "max_concurrent_requests", None),
@@ -1482,12 +1728,18 @@ class MCPServerManager:
                 delegate_server_ids = [
                     server.server_id
                     for server in self.get_registry().values()
-                    if getattr(server, "auth_type", None) == MCPAuth.oauth2
-                    and getattr(server, "delegate_auth_to_upstream", False) is True
-                    # M2M servers must not be exposed anonymously: an
-                    # unauthenticated caller would get LiteLLM to proxy tool
-                    # calls using its stored client_credentials.
-                    and not server.has_client_credentials
+                    if (
+                        getattr(server, "auth_type", None) == MCPAuth.oauth2
+                        and getattr(server, "delegate_auth_to_upstream", False) is True
+                        # M2M servers must not be exposed anonymously: an
+                        # unauthenticated caller would get LiteLLM to proxy tool
+                        # calls using its stored client_credentials. Resolve the flow
+                        # rather than reading has_client_credentials so an unstamped
+                        # M2M-shape row (null column, verbatim-read as non-M2M) still
+                        # fails closed here, matching the anonymous-delegate auth gate.
+                        and MCPServerManager.effective_oauth2_flow(server) != "client_credentials"
+                    )
+                    or getattr(server, "auth_type", None) == MCPAuth.true_passthrough
                 ]
                 combined_servers.update(delegate_server_ids)
 
@@ -2085,16 +2337,17 @@ class MCPServerManager:
         spec = None if transport == MCPTransport.stdio else to_server_spec(server)
         provider = cred_provider or self._cred_provider
         # A caller-supplied per-request override (mcp_auth_header / x-mcp-*) defers to the v1 path
-        # so it wins - except for the per-user modes the v2 resolver owns (authorization_code's
-        # stored token and token_exchange's RFC 8693 minted token). A caller must not be able to
-        # substitute another user's stored credential, nor silently disable the OBO exchange and
-        # forward an arbitrary bearer upstream, so we keep the v2 spec and ignore the override for
-        # both; the REST tools preview supplies its not-yet-persisted token through the resolver
-        # (cred_provider), never this path.
+        # so it wins - except for the modes the v2 resolver owns per-caller (authorization_code's
+        # stored token, token_exchange's RFC 8693 minted token, and the passthrough modes'
+        # forwarded caller token). A caller must not be able to substitute another user's stored
+        # credential, nor silently disable the OBO exchange and forward an arbitrary bearer
+        # upstream, so we keep the v2 spec and ignore the override for these; the REST tools
+        # preview supplies its not-yet-persisted token through the resolver (cred_provider),
+        # never this path.
         if (
             spec is not None
             and mcp_auth_header
-            and not isinstance(spec.config, (AuthorizationCodeConfig, TokenExchangeConfig))
+            and not isinstance(spec.config, (AuthorizationCodeConfig, PassthroughConfig, TokenExchangeConfig))
         ):
             spec = None
         auth_value = (
@@ -2161,11 +2414,17 @@ class MCPServerManager:
             server_url = server.url or ""
 
             if spec is not None:
+                inbound_token = subject_token
+                if isinstance(spec.config, PassthroughConfig):
+                    inbound_token, extra_headers = _take_forwarded_authorization(extra_headers)
+                    per_server_token = _passthrough_token_from_mcp_auth_header(mcp_auth_header)
+                    if per_server_token is not None:
+                        inbound_token = per_server_token
                 resolved_auth, extra_headers = await self._resolve_v2_auth(
                     server=server,
                     spec=spec,
                     provider=provider,
-                    subject_token=subject_token,
+                    subject_token=inbound_token,
                     user_api_key_auth=user_api_key_auth,
                     extra_headers=extra_headers,
                 )
@@ -3450,10 +3709,11 @@ class MCPServerManager:
         limit = mcp_server.max_concurrent_requests
         if limit is None or limit <= 0:
             return None
-        semaphore = self._server_call_semaphores.get(mcp_server.server_id)
-        if semaphore is None:
-            semaphore = asyncio.Semaphore(limit)
-            self._server_call_semaphores[mcp_server.server_id] = semaphore
+        cached = self._server_call_semaphores.get(mcp_server.server_id)
+        if cached is not None and cached[0] == limit:
+            return cached[1]
+        semaphore = asyncio.Semaphore(limit)
+        self._server_call_semaphores[mcp_server.server_id] = (limit, semaphore)
         return semaphore
 
     @asynccontextmanager
@@ -3586,6 +3846,13 @@ class MCPServerManager:
                     user_api_key_auth=user_api_key_auth,
                 ):
                     extra_headers = _without_authorization(extra_headers)
+        elif mcp_server.is_true_passthrough or mcp_server.is_oauth_delegate:
+            extra_headers = _client_forwarded_authorization_headers(
+                mcp_server=mcp_server,
+                oauth2_headers=oauth2_headers,
+                raw_headers=raw_headers,
+                user_api_key_auth=user_api_key_auth,
+            )
 
         if mcp_server.extra_headers and raw_headers:
             if extra_headers is None:
@@ -3667,17 +3934,21 @@ class MCPServerManager:
             # OBO: the exchanged token may have been revoked/rotated upstream since it was cached, so
             # an upstream 401 gets one re-mint + retry. Gated to this mode; all others keep the plain
             # single call below.
-            tool_call_coro = self._obo_call_tool_with_retry(
-                client=client,
-                call_tool_params=call_tool_params,
-                host_progress_callback=host_progress_callback,
-                mcp_server=mcp_server,
-                server_auth_header=server_auth_header,
-                extra_headers=extra_headers,
-                stdio_env=stdio_env,
-                subject_token=subject_token,
-                user_api_key_auth=user_api_key_auth,
-            )
+            async def _obo_call_tool_limited():
+                async with self._limit_outbound_concurrency(mcp_server):
+                    return await self._obo_call_tool_with_retry(
+                        client=client,
+                        call_tool_params=call_tool_params,
+                        host_progress_callback=host_progress_callback,
+                        mcp_server=mcp_server,
+                        server_auth_header=server_auth_header,
+                        extra_headers=extra_headers,
+                        stdio_env=stdio_env,
+                        subject_token=subject_token,
+                        user_api_key_auth=user_api_key_auth,
+                    )
+
+            tool_call_coro = _obo_call_tool_limited()
         else:
 
             async def _call_tool_via_client(client, params):
@@ -3771,6 +4042,19 @@ class MCPServerManager:
             return False
         return await self._cred_provider.has_user_token(to_subject(user_api_key_auth, None), spec)
 
+    async def invalidate_user_oauth_token_cache(self, user_id: str, server_id: str) -> None:
+        """Drop the v2 chain's cached token for ``(user_id, server_id)`` after the credential row
+        changes (re-auth, revoke), so the next resolve reads the new row instead of serving the
+        replaced token until its cache TTL. Best-effort: a cache-drop failure is logged, never
+        raised, because the DB write already succeeded and the TTL remains the backstop.
+        """
+        try:
+            await self._per_user_oauth_token_store.invalidate(user_id, server_id)
+        except Exception as exc:  # noqa: BLE001 - cache drop is best-effort; TTL is the backstop
+            verbose_logger.warning(
+                "Failed to invalidate cached MCP OAuth token for user=%s server=%s: %s", user_id, server_id, exc
+            )
+
     async def _resolve_oauth2_headers_for_tool_call(
         self,
         mcp_server: MCPServer,
@@ -3861,6 +4145,15 @@ class MCPServerManager:
         start_time = datetime.datetime.now()
         mcp_server = self._resolve_mcp_server_for_tool_call(server_name, name)
 
+        # Resolved before any hook runs so a missing BYOK credential (401) never
+        # leaves during-hook side effects (audit logging, rate-limit bookkeeping)
+        # recorded against a call that ultimately fails.
+        mcp_auth_header = await _resolve_byok_mcp_auth_header(
+            mcp_server,
+            user_api_key_auth,
+            mcp_auth_header,
+        )
+
         #########################################################
         # Pre MCP Tool Call Hook
         # Allow validation and modification of tool calls before execution
@@ -3907,9 +4200,25 @@ class MCPServerManager:
                     server_name,
                 )
 
+            auth_header_value = (
+                _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
+            )
+            forwarded_headers = _openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth)
+
             async def _call_openapi_via_handler():
-                async with self._limit_outbound_concurrency(mcp_server):
-                    return await self._call_openapi_tool_handler(mcp_server, name, arguments)
+                from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
+                    _request_auth_header,
+                    _request_extra_headers,
+                )
+
+                auth_token = _request_auth_header.set(auth_header_value)
+                extra_token = _request_extra_headers.set(forwarded_headers)
+                try:
+                    async with self._limit_outbound_concurrency(mcp_server):
+                        return await self._call_openapi_tool_handler(mcp_server, name, arguments)
+                finally:
+                    _request_auth_header.reset(auth_token)
+                    _request_extra_headers.reset(extra_token)
 
             tasks.append(asyncio.create_task(_call_openapi_via_handler()))
         else:
@@ -4465,6 +4774,11 @@ class MCPServerManager:
             authorization_url=server.authorization_url,
             token_url=server.token_url,
             registration_url=server.registration_url,
+            oauth2_flow=server.oauth2_flow,
+            token_exchange_endpoint=server.token_exchange_endpoint,
+            audience=server.audience,
+            subject_token_type=server.subject_token_type,
+            token_exchange_profile=server.token_exchange_profile,
             allow_all_keys=server.allow_all_keys,
             instructions=server.instructions,
             timeout=server.timeout,
@@ -4553,6 +4867,8 @@ class MCPServerManager:
             teams=[],
             mcp_access_groups=server.access_groups or [],
             allowed_tools=server.allowed_tools or [],
+            tool_name_to_display_name=server.tool_name_to_display_name,
+            tool_name_to_description=server.tool_name_to_description,
             extra_headers=server.extra_headers or [],
             mcp_info=server.mcp_info,
             static_headers=server.static_headers,
@@ -4566,6 +4882,11 @@ class MCPServerManager:
             authorization_url=server.authorization_url,
             token_url=server.token_url,
             registration_url=server.registration_url,
+            oauth2_flow=server.oauth2_flow,
+            token_exchange_endpoint=server.token_exchange_endpoint,
+            audience=server.audience,
+            subject_token_type=server.subject_token_type,
+            token_exchange_profile=server.token_exchange_profile,
             allow_all_keys=server.allow_all_keys,
             available_on_public_internet=server.available_on_public_internet,
             delegate_auth_to_upstream=server.delegate_auth_to_upstream,

litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py

@@ -23,12 +23,13 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.types import (
     AuthorizationCodeConfig,
     CredError,
     NoneConfig,
+    PassthroughConfig,
     ServerSpec,
     SharedKey,
     Subject,
     TokenExchangeConfig,
 )
-from litellm.types.mcp import MCPAuth
+from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE, MCPAuth
 
 if TYPE_CHECKING:
     from litellm.proxy._types import UserAPIKeyAuth
@@ -62,9 +63,10 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]:
     an ``assert_never`` tail, so a newly added auth mode fails the type gate here until it is
     explicitly mapped or explicitly deferred, rather than silently falling through to v1. Live
     modes: ``none``, the static-header family (``api_key`` plus the Authorization schemes,
-    all shared-key), ``oauth2`` per-user tokens (``authorization_code``), and
-    ``oauth2_token_exchange`` (OBO); client_credentials (M2M), delegated/passthrough
-    oauth2, and SigV4 return None and stay on v1.
+    all shared-key), ``oauth2`` per-user tokens (``authorization_code``), ``oauth2_token_exchange``
+    (OBO), and the client-forwarded token modes ``true_passthrough`` / ``oauth_delegate``
+    (``PassthroughConfig``); client_credentials (M2M), delegated/passthrough oauth2, and SigV4
+    return None and stay on v1.
     """
     if server.is_byok:
         return None  # per-user BYOK source not migrated yet -> defer to v1 (any auth_type)
@@ -94,6 +96,8 @@ def to_server_spec(server: MCPServer) -> Optional[ServerSpec]:
                 )
             # client_credentials (M2M) and delegate/passthrough oauth2 stay on v1
             return None
+        case MCPAuth.true_passthrough | MCPAuth.oauth_delegate:
+            return ServerSpec(server_id=server.server_id, resource=resource, config=PassthroughConfig())
         case MCPAuth.oauth2_token_exchange:
             return _token_exchange_spec(server, resource)
         case MCPAuth.aws_sigv4:
@@ -124,7 +128,7 @@ def _token_exchange_spec(server: MCPServer, resource: str) -> Optional[ServerSpe
         resource=resource,
         config=TokenExchangeConfig(
             profile=profile,
-            subject_token_type=server.subject_token_type or "urn:ietf:params:oauth:token-type:access_token",
+            subject_token_type=server.subject_token_type or DEFAULT_SUBJECT_TOKEN_TYPE,
             token_exchange_endpoint=endpoint,
             audience=server.audience,
             client_id=server.client_id,

litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py

@@ -69,6 +69,17 @@ class OAuthTokenStore(Protocol):
     async def fetch(self, user_id: str, server_id: str) -> OAuthToken | None: ...
 
 
+class InvalidatableOAuthTokenStore(OAuthTokenStore, Protocol):
+    """An ``OAuthTokenStore`` whose cached entry for a ``(user, server)`` pair can be dropped.
+
+    The write side calls ``invalidate`` after a (re)authorization or revocation changes the
+    credential row, so reads stop serving the replaced token immediately instead of until its
+    cache TTL. ``CachedOAuthTokenStore`` (the top of the per-user chain) satisfies this.
+    """
+
+    async def invalidate(self, user_id: str, server_id: str) -> None: ...
+
+
 class TokenRefresher(Protocol):
     """Mints a fresh token from an expired one and persists it, returning the new token.
 

litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py

@@ -24,8 +24,8 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_toke
 )
 from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
     CachedOAuthTokenStore,
+    InvalidatableOAuthTokenStore,
     OAuthToken,
-    OAuthTokenStore,
     RefreshCoordinator,
     RefreshingTokenStore,
     TokenCacheBackend,
@@ -51,7 +51,7 @@ if TYPE_CHECKING:
 _DEFAULT_TTL_SECONDS = 300.0
 
 ServerLookup = Callable[[str], "MCPServer | None"]
-StoreBuilder = Callable[[ServerLookup], tuple[OAuthTokenStore, bool]]
+StoreBuilder = Callable[[ServerLookup], tuple[InvalidatableOAuthTokenStore, bool]]
 
 
 async def _read_credential(user_id: str, server_id: str) -> dict[str, object] | None:
@@ -185,7 +185,7 @@ class LazyPerUserOAuthTokenStore:
         self._server_lookup = server_lookup
         self._store_builder = store_builder
         self._redis_available = redis_available
-        self._store: OAuthTokenStore | None = None
+        self._store: InvalidatableOAuthTokenStore | None = None
         self._uses_redis = False
         self._fetch_lock = asyncio.Condition()
         self._local_fetches = 0
@@ -203,7 +203,26 @@ class LazyPerUserOAuthTokenStore:
             if not uses_redis:
                 await self._finish_local_fetch()
 
-    async def _store_for_fetch(self) -> tuple[OAuthTokenStore, bool]:
+    async def invalidate(self, user_id: str, server_id: str) -> None:
+        """Drop the chain's cached entry for ``(user_id, server_id)`` after the credential row
+        changes (re-auth, revoke). Builds the chain if no fetch has run yet, so a shared (Redis)
+        cache entry written by another worker is dropped too; the in-process case is then a no-op
+        on an empty cache.
+        """
+        if self._uses_redis:
+            store = self._store
+            if store is not None:
+                await store.invalidate(user_id, server_id)
+                return
+
+        store, uses_redis = await self._store_for_fetch()
+        try:
+            await store.invalidate(user_id, server_id)
+        finally:
+            if not uses_redis:
+                await self._finish_local_fetch()
+
+    async def _store_for_fetch(self) -> tuple[InvalidatableOAuthTokenStore, bool]:
         async with self._fetch_lock:
             while (
                 self._store is not None and not self._uses_redis and self._redis_available() and self._local_fetches > 0

litellm/proxy/_experimental/mcp_server/outbound_credentials/resolver.py

@@ -7,10 +7,11 @@ no precedence cascade. It is wildcard-free with an `assert_never` tail, so addin
 an arm fails the type gate (basedpyright `reportMatchNotExhaustive`); a bypassed gate fails loudly
 at runtime instead of returning `None`.
 
-`none` and `api_key` (shared-key source) are live, as is `authorization_code`, which reads the
-user's token from the injected `OAuthTokenStore`, and `token_exchange`, which swaps the caller's
-inbound token through the injected `TokenExchanger`. The remaining arms are `not_implemented` stubs
-that each land in a follow-up PR with their seam. Pure v2: no imports from v1.
+`none`, `api_key` (shared-key source), and `passthrough` (forwards the caller's own inbound token)
+are live, as is `authorization_code`, which reads the user's token from the injected
+`OAuthTokenStore`, and `token_exchange`, which swaps the caller's inbound token through the
+injected `TokenExchanger`. The remaining arms are `not_implemented` stubs that each land in a
+follow-up PR with their seam. Pure v2: no imports from v1.
 """
 
 from __future__ import annotations
@@ -97,7 +98,7 @@ class UpstreamCredentialProvider:
             case ApiKeyConfig() as config:
                 return self._api_key(config)
             case PassthroughConfig():
-                return _not_implemented(AuthSpecKind.passthrough)
+                return self._passthrough(subject)
             case ClientCredentialsConfig():
                 return _not_implemented(AuthSpecKind.client_credentials)
             case TokenExchangeConfig() as config:
@@ -118,6 +119,18 @@ class UpstreamCredentialProvider:
         """
         return await self._authz_token(subject, server) is not None
 
+    def _passthrough(self, subject: Subject) -> Result[httpx.Auth, CredError]:
+        """Forward the caller's own upstream credential verbatim; the gateway mints nothing.
+
+        The inbound token is the caller's already-disambiguated ``Authorization`` (never the LiteLLM
+        admission credential; the edge adapter drops that before building the ``Subject``). When it is
+        absent the request is sent unauthenticated so the upstream's own 401 surfaces, rather than the
+        gateway challenging on the upstream's behalf.
+        """
+        if subject.inbound_token is None:
+            return Ok(NoOpAuth())
+        return Ok(StaticHeaderAuth(subject.inbound_token.get_secret_value(), header_name="Authorization"))
+
     def _api_key(self, config: ApiKeyConfig) -> Result[httpx.Auth, CredError]:
         match config.key_source:
             case SharedKey() as source:

litellm/proxy/_experimental/mcp_server/outbound_credentials/types.py

@@ -39,6 +39,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.result import (
     Ok,
     Result,
 )
+from litellm.types.mcp import DEFAULT_SUBJECT_TOKEN_TYPE
 
 
 class AuthSpecKind(str, Enum):
@@ -215,7 +216,7 @@ class TokenExchangeConfig(BaseModel):
     model_config = ConfigDict(frozen=True)
     kind: Literal[AuthSpecKind.token_exchange] = AuthSpecKind.token_exchange
     profile: Literal["rfc8693", "entra_obo"] = "rfc8693"
-    subject_token_type: str = "urn:ietf:params:oauth:token-type:access_token"
+    subject_token_type: str = DEFAULT_SUBJECT_TOKEN_TYPE
     token_exchange_endpoint: str | None = None
     audience: str | None = None
     client_id: str | None = None

litellm/proxy/_experimental/mcp_server/rest_endpoints.py

@@ -8,6 +8,7 @@ from typing import (
     Dict,
     List,
     Literal,
+    Mapping,
     Optional,
     Set,
     Tuple,
@@ -77,7 +78,8 @@ if MCP_AVAILABLE:
         ListMCPToolsRestAPIResponseObject,
         MCPInfo,
         MCPServer,
-        _fire_mcp_success_logging,
+        _apply_toolset_scope,
+        _fire_mcp_tool_call_logging,
         _tool_name_matches,
         execute_mcp_tool,
         filter_tools_by_allowed_tools,
@@ -85,23 +87,32 @@ if MCP_AVAILABLE:
 
     ########################################################
     ############ MCP Server REST API Routes #################
-    async def _safe_fire_mcp_success_logging(
+    async def _safe_fire_mcp_tool_call_logging(
         logging_obj: Optional[Any],
         result: Any,
         start_time: datetime,
         end_time: datetime,
+        user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        request_data: Optional[Mapping[str, object]] = None,
     ) -> None:
         if logging_obj is None:
             return
         logging_results = await asyncio.gather(
-            _fire_mcp_success_logging(logging_obj, result, start_time, end_time),
+            _fire_mcp_tool_call_logging(
+                logging_obj,
+                result,
+                start_time,
+                end_time,
+                user_api_key_auth=user_api_key_auth,
+                request_data=request_data,
+            ),
             return_exceptions=True,
         )
         logging_error = logging_results[0]
         if isinstance(logging_error, asyncio.CancelledError):
             raise logging_error
         if isinstance(logging_error, BaseException):
-            verbose_logger.warning("MCP tool success logging failed (continuing): %s", logging_error)
+            verbose_logger.warning("MCP tool call logging failed (continuing): %s", logging_error)
 
     def _get_server_auth_header(
         server,
@@ -541,10 +552,37 @@ if MCP_AVAILABLE:
             "message": "Successfully retrieved tools",
         }
 
+    def _as_query_str(value: Any) -> Optional[str]:
+        """Coerce an Optional[str] Query param to str|None, dropping unresolved FastAPI defaults."""
+        return value if isinstance(value, str) else None
+
+    async def _resolve_toolset_scope(
+        toolset_name: Optional[str],
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> UserAPIKeyAuth:
+        """Resolve ``toolset_name`` to its scoped ``UserAPIKeyAuth``, or return unchanged."""
+        if not toolset_name:
+            return user_api_key_dict
+
+        from litellm.proxy.utils import get_prisma_client_or_throw
+
+        prisma_client = get_prisma_client_or_throw("Database not available. Connect a database to your proxy")
+        toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, toolset_name)
+        if toolset is None:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Toolset '{toolset_name}' not found",
+            )
+        return await _apply_toolset_scope(user_api_key_dict, toolset.toolset_id)
+
     @router.get("/tools/list", dependencies=[Depends(user_api_key_auth)])
     async def list_tool_rest_api(
         request: Request,
         server_id: Optional[str] = Query(None, description="The server id to list tools for"),
+        mcp_server_name: Optional[str] = Query(
+            None, description="Filter tools to a single MCP server by name or alias"
+        ),
+        toolset_name: Optional[str] = Query(None, description="Filter tools to a single toolset by name"),
         include_disabled_tools: bool = Query(
             False,
             description=(
@@ -582,16 +620,29 @@ if MCP_AVAILABLE:
         )
 
         try:
+            mcp_server_name = _as_query_str(mcp_server_name)
+            toolset_name = _as_query_str(toolset_name)
+
             # The full catalog (allowlist filter skipped) is admin-only so the
             # REST endpoint can't be used to enumerate deliberately-disabled tools.
             apply_tool_filters = not (
                 include_disabled_tools and user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
             )
 
-            if apply_tool_filters and getattr(
-                getattr(user_api_key_dict, "object_permission", None),
-                "mcp_tool_search_enabled",
-                False,
+            user_api_key_dict = await _resolve_toolset_scope(toolset_name, user_api_key_dict)
+
+            if server_id is None:
+                server_id = mcp_server_name
+
+            if (
+                apply_tool_filters
+                and server_id is None
+                and toolset_name is None
+                and getattr(
+                    getattr(user_api_key_dict, "object_permission", None),
+                    "mcp_tool_search_enabled",
+                    False,
+                )
             ):
                 from litellm.proxy._experimental.mcp_server.tool_search import (
                     get_virtual_tool_definitions,
@@ -719,6 +770,8 @@ if MCP_AVAILABLE:
                 request_path=request.scope.get("_original_path") or request.url.path,
             )
         except HTTPException as http_exc:
+            if http_exc.status_code == status.HTTP_404_NOT_FOUND:
+                raise
             # Internal access/IP 403s keep the legacy error-dict response shape
             # so the existing contract stays intact.
             verbose_logger.exception("HTTPException in list_tool_rest_api: %s", str(http_exc))
@@ -829,7 +882,14 @@ if MCP_AVAILABLE:
                         raw_headers=virtual_raw_headers,
                         litellm_logging_obj=virtual_logging_obj,
                     )
-                    await _safe_fire_mcp_success_logging(virtual_logging_obj, result, _tool_start_time, datetime.now())
+                    await _safe_fire_mcp_tool_call_logging(
+                        virtual_logging_obj,
+                        result,
+                        _tool_start_time,
+                        datetime.now(),
+                        user_api_key_auth=user_api_key_dict,
+                        request_data=data,
+                    )
                     return result
 
             # Validate required parameters early
@@ -912,7 +972,14 @@ if MCP_AVAILABLE:
                 litellm_logging_obj=data.get("litellm_logging_obj"),
                 requested_server_id=canonical_server_id,
             )
-            await _safe_fire_mcp_success_logging(logging_obj, result, _tool_start_time, datetime.now())
+            await _safe_fire_mcp_tool_call_logging(
+                logging_obj,
+                result,
+                _tool_start_time,
+                datetime.now(),
+                user_api_key_auth=user_api_key_dict,
+                request_data=data,
+            )
             return result
         except MCPMissingUserEnvVarsError as e:
             verbose_logger.info(

litellm/proxy/_experimental/mcp_server/server.py

@@ -20,6 +20,7 @@ from typing import (
     Callable,
     Dict,
     List,
+    Mapping,
     Optional,
     Set,
     Tuple,
@@ -47,7 +48,10 @@ from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
 from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
     get_request_base_url,
 )
-from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError
+from litellm.proxy._experimental.mcp_server.exceptions import (
+    MCPToolResultError,
+    MCPUpstreamAuthError,
+)
 from litellm.proxy._experimental.mcp_server.mcp_context import (
     _mcp_active_toolset_id,
     _mcp_gateway_initialize_instructions,
@@ -60,6 +64,7 @@ from litellm.proxy._experimental.mcp_server.utils import (
     LITELLM_MCP_SERVER_VERSION,
     MCPMissingUserEnvVarsError,
     add_server_prefix_to_name,
+    extract_mcp_tool_result_error_message,
     get_server_prefix,
     iter_known_server_prefixes,
 )
@@ -331,6 +336,8 @@ if MCP_AVAILABLE:
     )
     from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
         MCPServerManager,
+        _caller_authorization_fans_out,
+        _client_forwarded_authorization_headers,
         _should_strip_caller_authorization,
         _without_authorization,
         global_mcp_server_manager,
@@ -716,7 +723,7 @@ if MCP_AVAILABLE:
         if not (host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta):
             return None
         host_token = getattr(host_ctx.meta, "progressToken", None)
-        if not (host_token and hasattr(host_ctx, "session") and host_ctx.session):
+        if host_token is None or not (hasattr(host_ctx, "session") and host_ctx.session):
             return None
         host_session = host_ctx.session
 
@@ -732,7 +739,7 @@ if MCP_AVAILABLE:
             except Exception as e:
                 verbose_logger.error(f"Failed to forward progress to Host: {e}")
 
-        verbose_logger.debug(f"Host progressToken captured: {host_token[:8]}...")
+        verbose_logger.debug(f"Host progressToken captured: {str(host_token)[:8]}...")
         return forward_progress
 
     async def _build_virtual_call_logging_obj(
@@ -1427,18 +1434,8 @@ if MCP_AVAILABLE:
         for allowed_mcp_server_id in allowed_mcp_server_ids:
             mcp_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id)
             if mcp_server is not None:
-                # Apply oauth2_flow resolution for legacy DB rows where it may be NULL
-                resolved_flow = MCPServerManager._resolve_oauth2_flow(
-                    auth_type=mcp_server.auth_type,
-                    oauth2_flow=mcp_server.oauth2_flow,
-                    token_url=mcp_server.token_url,
-                    authorization_url=mcp_server.authorization_url,
-                    client_id=mcp_server.client_id,
-                    client_secret=mcp_server.client_secret,
-                )
-                if resolved_flow and resolved_flow != mcp_server.oauth2_flow:
-                    # Create a new instance with the resolved flow for this request
-                    mcp_server = mcp_server.model_copy(update={"oauth2_flow": resolved_flow})
+                # Apply the request-time oauth2_flow backstop for legacy null rows.
+                mcp_server = MCPServerManager.resolve_oauth2_flow_for_request(mcp_server)
                 allowed_mcp_servers.append(mcp_server)
 
         if mcp_servers is not None:
@@ -1449,6 +1446,35 @@ if MCP_AVAILABLE:
 
         return allowed_mcp_servers
 
+    def _client_has_per_server_auth_header(
+        server: MCPServer,
+        mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]],
+    ) -> bool:
+        """True if the request carries a per-server ``x-mcp-{alias}-authorization``
+        header for this server. This is the multi-server binding: it names one
+        upstream, so it is unambiguously the caller's upstream token regardless of
+        auth mode (never the LiteLLM admission credential).
+
+        Resolves through the same ``lookup_mcp_server_auth_in_headers`` egress uses, so
+        the connect gate and egress agree on which per-server header names match: a
+        dashboard client sends ``x-mcp-{sanitize_mcp_alias_for_header(alias)}-authorization``,
+        and matching only the raw alias here would 401 a token egress would forward.
+        """
+        if not mcp_server_auth_headers:
+            return False
+        from litellm.proxy._experimental.mcp_server.utils import (
+            lookup_mcp_server_auth_in_headers,
+        )
+
+        server_headers = lookup_mcp_server_auth_in_headers(
+            mcp_server_auth_headers, alias=server.alias, server_name=server.server_name
+        )
+        if isinstance(server_headers, str):
+            return bool(server_headers.strip())
+        if isinstance(server_headers, dict):
+            return any(isinstance(hk, str) and hk.lower() == "authorization" for hk in server_headers)
+        return False
+
     def _client_has_passthrough_authorization(
         server: MCPServer,
         oauth2_headers: Optional[Dict[str, str]],
@@ -1466,24 +1492,7 @@ if MCP_AVAILABLE:
             for k in oauth2_headers.keys():
                 if k.lower() == "authorization":
                     return True
-        if mcp_server_auth_headers:
-            for key in (server.alias, server.server_name, server.name):
-                if not key:
-                    continue
-                server_headers = None
-                for k, v in mcp_server_auth_headers.items():
-                    if k.lower() == key.lower():
-                        server_headers = v
-                        break
-                if server_headers is None:
-                    continue
-                if isinstance(server_headers, str) and server_headers.strip():
-                    return True
-                if isinstance(server_headers, dict):
-                    for hk in server_headers.keys():
-                        if hk.lower() == "authorization":
-                            return True
-        return False
+        return _client_has_per_server_auth_header(server, mcp_server_auth_headers)
 
     async def _get_user_oauth_extra_headers_from_db(
         server: MCPServer,
@@ -1538,8 +1547,16 @@ if MCP_AVAILABLE:
         oauth2_headers: Optional[Dict[str, str]],
         raw_headers: Optional[Dict[str, str]],
         user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        scope_servers: Optional[list[MCPServer]] = None,
     ) -> Tuple[Optional[Union[Dict[str, str], str]], Optional[Dict[str, str]]]:
-        """Build auth and extra headers for a server."""
+        """Build auth and extra headers for a server.
+
+        ``scope_servers`` is the full server list a fan-out handler iterates. Passing it lets the
+        client-forwarded token modes withhold the caller's request-wide ``Authorization`` when
+        another server in the scope would also receive it (``_caller_authorization_fans_out``);
+        explicitly-addressed operations leave it None. Per-server ``x-mcp-{alias}-authorization``
+        headers are unaffected — they bind one token to one server and are the multi-server shape.
+        """
         server_auth_header: Optional[Union[Dict[str, str], str]] = None
         if mcp_server_auth_headers:
             from litellm.proxy._experimental.mcp_server.utils import (
@@ -1553,6 +1570,16 @@ if MCP_AVAILABLE:
             )
 
         extra_headers: Optional[Dict[str, str]] = None
+        is_client_forwarded_mode = server.is_true_passthrough or server.is_oauth_delegate
+        # In a multi-server listing scope the request-wide Authorization can only carry one token,
+        # so it is withheld from a client-forwarded server when another server in scope also consumes
+        # it (RFC 9700 cross-resource replay); such scopes must bind per-server via
+        # x-mcp-{alias}-authorization. The decision is computed once so BOTH the forwarding branch and
+        # the extra_headers copy loop below honor it — otherwise a server that lists Authorization in
+        # extra_headers would re-copy the withheld bearer from raw_headers and replay it anyway.
+        withhold_forwarded_authorization = is_client_forwarded_mode and _caller_authorization_fans_out(
+            server, scope_servers
+        )
         if server.auth_type == MCPAuth.oauth2:
             # For OAuth2 M2M servers, upstream Authorization must come from
             # client_credentials token fetch, never from caller headers.
@@ -1571,6 +1598,14 @@ if MCP_AVAILABLE:
                     user_api_key_auth=user_api_key_auth,
                 ):
                     extra_headers = _without_authorization(extra_headers)
+        elif is_client_forwarded_mode:
+            if not withhold_forwarded_authorization:
+                extra_headers = _client_forwarded_authorization_headers(
+                    mcp_server=server,
+                    oauth2_headers=oauth2_headers,
+                    raw_headers=raw_headers,
+                    user_api_key_auth=user_api_key_auth,
+                )
 
         if server.extra_headers and raw_headers:
             if extra_headers is None:
@@ -1591,7 +1626,9 @@ if MCP_AVAILABLE:
             for header in server.extra_headers:
                 if not isinstance(header, str):
                     continue
-                if header.lower() == "authorization" and strip_caller_authorization:
+                if header.lower() == "authorization" and (
+                    strip_caller_authorization or withhold_forwarded_authorization
+                ):
                     continue
                 header_value = normalized_raw_headers.get(header.lower())
                 if header_value is None:
@@ -1795,6 +1832,7 @@ if MCP_AVAILABLE:
                     oauth2_headers=oauth2_headers,
                     raw_headers=raw_headers,
                     user_api_key_auth=user_api_key_auth,
+                    scope_servers=allowed_mcp_servers,
                 )
 
                 # Prefer server-stored per-user OAuth when configured, so a stale
@@ -1981,6 +2019,7 @@ if MCP_AVAILABLE:
                 oauth2_headers=oauth2_headers,
                 raw_headers=raw_headers,
                 user_api_key_auth=user_api_key_auth,
+                scope_servers=allowed_mcp_servers,
             )
 
             try:
@@ -2033,6 +2072,7 @@ if MCP_AVAILABLE:
                 oauth2_headers=oauth2_headers,
                 raw_headers=raw_headers,
                 user_api_key_auth=user_api_key_auth,
+                scope_servers=allowed_mcp_servers,
             )
 
             try:
@@ -2083,6 +2123,7 @@ if MCP_AVAILABLE:
                 oauth2_headers=oauth2_headers,
                 raw_headers=raw_headers,
                 user_api_key_auth=user_api_key_auth,
+                scope_servers=allowed_mcp_servers,
             )
 
             try:
@@ -2753,12 +2794,40 @@ if MCP_AVAILABLE:
 
         return response
 
-    async def _fire_mcp_success_logging(
+    _MCP_CREDENTIAL_REQUEST_FIELDS = frozenset(
+        {
+            "raw_headers",
+            "mcp_auth_header",
+            "mcp_server_auth_headers",
+            "oauth2_headers",
+            "user_api_key_auth",
+        }
+    )
+
+    async def _fire_mcp_tool_call_logging(
         logging_obj: LiteLLMLoggingObj,
         result: Any,
         start_time: datetime,
         end_time: datetime,
+        user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        request_data: Optional[Mapping[str, object]] = None,
     ) -> None:
+        """Fire post-call logging for an executed MCP tool call.
+
+        A result with ``isError=True`` is logged as a failure (``status="failure"``
+        payload, so OTel marks the span ERROR) while the HTTP wire behavior stays
+        200 + ``isError: true`` per the MCP spec. The error check runs after
+        ``async_post_mcp_tool_call_hook`` because guardrails may flip the result
+        to ``isError=True`` in that hook. Raised exceptions never reach here (the
+        ``@client`` wrapper and ``call_mcp_tool``'s except path log those), so
+        this cannot double-log a failure.
+
+        ``request_data`` may carry credential-bearing fields (the REST path puts
+        ``raw_headers``, ``mcp_auth_header``, ``mcp_server_auth_headers``, and
+        ``oauth2_headers`` at the top level of its data dict), so those are
+        stripped before the dict is handed to ``post_call_failure_hook``
+        callbacks.
+        """
         logging_obj.post_call(original_response=result)
         await logging_obj.async_post_mcp_tool_call_hook(
             kwargs=logging_obj.model_call_details,
@@ -2767,7 +2836,31 @@ if MCP_AVAILABLE:
             end_time=end_time,
         )
         logging_obj.call_type = CallTypes.call_mcp_tool.value
-        await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time)
+        error_message = extract_mcp_tool_result_error_message(result)
+        if error_message is None:
+            await logging_obj.async_success_handler(result=result, start_time=start_time, end_time=end_time)
+            return
+
+        logging_obj.has_run_logging(event_type="sync_success")
+        logging_obj.has_run_logging(event_type="async_success")
+        tool_error = MCPToolResultError(error_message)
+        logging_obj.failure_handler(tool_error, "", start_time, end_time)
+        await logging_obj.async_failure_handler(tool_error, "", start_time, end_time)
+
+        if user_api_key_auth is None:
+            return
+        from litellm.proxy.proxy_server import proxy_logging_obj
+
+        if proxy_logging_obj:
+            sanitized_request_data = {
+                key: value for key, value in (request_data or {}).items() if key not in _MCP_CREDENTIAL_REQUEST_FIELDS
+            }
+            await proxy_logging_obj.post_call_failure_hook(
+                request_data=sanitized_request_data,
+                original_exception=tool_error,
+                user_api_key_dict=user_api_key_auth,
+                route="/mcp/call_tool",
+            )
 
     @client
     async def call_mcp_tool(
@@ -2800,6 +2893,9 @@ if MCP_AVAILABLE:
             for allowed_mcp_server_id in allowed_mcp_server_ids:
                 allowed_server = global_mcp_server_manager.get_mcp_server_by_id(allowed_mcp_server_id)
                 if allowed_server is not None:
+                    # Same request-time oauth2_flow backstop the listing path applies,
+                    # so a null-flow M2M-shape row is treated as M2M on tool calls too.
+                    allowed_server = MCPServerManager.resolve_oauth2_flow_for_request(allowed_server)
                     allowed_mcp_servers.append(allowed_server)
 
             allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
@@ -2840,7 +2936,14 @@ if MCP_AVAILABLE:
             raise
 
         if litellm_logging_obj:
-            await _fire_mcp_success_logging(litellm_logging_obj, response, start_time, datetime.now())
+            await _fire_mcp_tool_call_logging(
+                logging_obj=litellm_logging_obj,
+                result=response,
+                start_time=start_time,
+                end_time=datetime.now(),
+                user_api_key_auth=user_api_key_auth,
+                request_data=kwargs,
+            )
         return response
 
     async def mcp_get_prompt(
@@ -3525,6 +3628,41 @@ if MCP_AVAILABLE:
                     headers={"www-authenticate": www_authenticate},
                 )
 
+            if (
+                server
+                and server.is_oauth_delegate
+                and len(mcp_servers or []) == 1
+                and _get_forwarded_auth_from_scope(scope) is None
+                and not _client_has_per_server_auth_header(server, mcp_server_auth_headers)
+            ):
+                www_authenticate = _get_passthrough_www_authenticate(
+                    scope=scope,
+                    server_name=server_name,
+                )
+                raise HTTPException(
+                    status_code=401,
+                    detail="Unauthorized",
+                    headers={"www-authenticate": www_authenticate},
+                )
+
+            if (
+                server
+                and server.is_true_passthrough
+                and len(mcp_servers or []) == 1
+                and not _scope_has_authorization_header(scope)
+                and not _client_has_per_server_auth_header(server, mcp_server_auth_headers)
+            ):
+                upstream_status, upstream_www_authenticate = await _probe_upstream_auth(server.url or "", "")
+                if upstream_status == 401 and upstream_www_authenticate:
+                    raise HTTPException(
+                        status_code=401,
+                        detail="Unauthorized",
+                        headers={"www-authenticate": upstream_www_authenticate},
+                    )
+
+    def _scope_has_authorization_header(scope: Scope) -> bool:
+        return any(key.lower() == b"authorization" for key, _ in scope.get("headers", []))
+
     def _get_forwarded_auth_from_scope(scope: Scope) -> Optional[str]:
         """Return the upstream-bound ``Authorization`` header value, or None.
 
@@ -3552,7 +3690,7 @@ if MCP_AVAILABLE:
         url: str,
         auth_header: str,
         timeout: float = 5.0,
-    ) -> tuple:
+    ) -> tuple[int, Optional[str]]:
         """JSON-RPC initialize-probe the upstream URL to check whether the token is accepted.
 
         Uses POST so StreamableHTTP MCP servers run the same auth path as a
@@ -3582,8 +3720,8 @@ if MCP_AVAILABLE:
             },
         }
         probe_headers = {
-            "Authorization": auth_header,
             "Accept": "application/json, text/event-stream",
+            **({"Authorization": auth_header} if auth_header else {}),
         }
         try:
             resp = await client.post(

litellm/proxy/_experimental/mcp_server/utils.py

@@ -214,12 +214,14 @@ def server_applies_tool_allowlist(mcp_server: Any) -> bool:
 
 def validate_and_normalize_mcp_server_payload(payload: Any) -> None:
     """
-    Validate and normalize MCP server payload fields (server_name and alias).
+    Validate and normalize MCP server payload fields (server_name, alias, and
+    tool_name_to_display_name).
 
     This function:
     1. Validates that server_name and alias don't contain the MCP_TOOL_PREFIX_SEPARATOR
-    2. Normalizes alias by replacing spaces with underscores
-    3. Sets default alias if not provided (using server_name as base)
+    2. Validates that tool_name_to_display_name values satisfy Bedrock's tool-name pattern
+    3. Normalizes alias by replacing spaces with underscores
+    4. Sets default alias if not provided (using server_name as base)
 
     Args:
         payload: The payload object containing server_name and alias fields
@@ -235,6 +237,10 @@ def validate_and_normalize_mcp_server_payload(payload: Any) -> None:
     if hasattr(payload, "alias") and payload.alias:
         validate_mcp_server_name(payload.alias, raise_http_exception=True)
 
+    # Tool display name validation: must satisfy Bedrock's tool-name pattern
+    if hasattr(payload, "tool_name_to_display_name") and payload.tool_name_to_display_name:
+        validate_tool_display_names(payload.tool_name_to_display_name)
+
     # Alias normalization and defaulting
     alias = getattr(payload, "alias", None)
     server_name = getattr(payload, "server_name", None)
@@ -409,6 +415,61 @@ def validate_mcp_server_name(server_name: str, raise_http_exception: bool = Fals
             raise Exception(error_message)
 
 
+def extract_mcp_tool_result_error_message(result: object) -> Optional[str]:
+    """The first text content of an ``isError=True`` tool result, or ``None``
+    when the result is not an error.
+
+    Accepts both ``mcp.types.CallToolResult`` objects and their dict
+    equivalents, duck-typed so the ``mcp`` package is not required.
+    """
+    is_error: object = result.get("isError") if isinstance(result, Mapping) else getattr(result, "isError", None)
+    if is_error is not True:
+        return None
+    content: object = result.get("content") if isinstance(result, Mapping) else getattr(result, "content", None)
+    if isinstance(content, (list, tuple)):
+        for item in content:
+            text: object = item.get("text") if isinstance(item, Mapping) else getattr(item, "text", None)
+            if isinstance(text, str) and text:
+                return text
+    return "MCP tool call returned isError=true"
+
+
+TOOL_DISPLAY_NAME_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$")
+
+
+def validate_tool_display_names(tool_name_to_display_name: Optional[Mapping[str, str]]) -> None:
+    """
+    Validate tool display name overrides against Bedrock's tool-name constraint.
+
+    A display name replaces the tool name sent to the LLM provider, so it must
+    satisfy the strictest provider requirement in use (Bedrock's
+    ``[a-zA-Z0-9_-]+``); a name with spaces or other characters saves
+    successfully but fails every subsequent Bedrock tool call.
+
+    Raises:
+        HTTPException: If any display name fails the pattern.
+    """
+    if not tool_name_to_display_name:
+        return
+
+    for original_name, display_name in tool_name_to_display_name.items():
+        if display_name and not TOOL_DISPLAY_NAME_PATTERN.match(display_name):
+            from fastapi import HTTPException
+            from starlette import status
+
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail={
+                    "error": (
+                        f"Invalid display name '{display_name}' for tool '{original_name}'. "
+                        "Display names may only contain letters, digits, underscores, and "
+                        "hyphens (no spaces or other special characters), since they replace "
+                        "the tool name sent to the LLM provider."
+                    )
+                },
+            )
+
+
 class MCPMissingUserEnvVarsError(Exception):
     """Raised when an MCP request can't be built because the calling user has
     not supplied one or more required per-user environment variables.

litellm/proxy/_types.py

@@ -1045,6 +1045,7 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase):
     model_rpm_limit: Optional[dict] = None
     model_tpm_limit: Optional[dict] = None
     mcp_rpm_limit: Optional[Dict[str, int]] = None
+    tag_rpm_limit: Optional[dict[str, int]] = None
     guardrails: Optional[List[str]] = None
     policies: Optional[List[str]] = None
     prompts: Optional[List[str]] = None
@@ -1070,6 +1071,7 @@ class KeyRequestBase(GenerateRequestBase):
     budget_id: Optional[str] = None
     tags: Optional[List[str]] = None
     disable_global_guardrails: Optional[bool] = None
+    throttle_on_budget_exceeded: Optional[bool] = None
     enforced_params: Optional[List[str]] = None
     allowed_routes: Optional[list] = []
     allowed_passthrough_routes: Optional[list] = None
@@ -1254,6 +1256,14 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase):
     token_url: Optional[str] = None
     registration_url: Optional[str] = None
     oauth2_flow: Optional[Literal["client_credentials", "authorization_code"]] = None
+    # Token Exchange (OBO) fields — RFC 8693. These top-level fields are the
+    # canonical shape; the same keys inside ``credentials`` are the legacy
+    # pre-column REST shape and are lifted into these columns on write (an
+    # explicit top-level value wins) and stripped from the stored blob.
+    token_exchange_endpoint: Optional[str] = None
+    audience: Optional[str] = None
+    subject_token_type: Optional[str] = None
+    token_exchange_profile: Optional[str] = None
     allow_all_keys: bool = False
     available_on_public_internet: bool = True
     delegate_auth_to_upstream: bool = False
@@ -1340,6 +1350,14 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase):
     token_url: Optional[str] = None
     registration_url: Optional[str] = None
     oauth2_flow: Optional[Literal["client_credentials", "authorization_code"]] = None
+    # Token Exchange (OBO) fields — RFC 8693. These top-level fields are the
+    # canonical shape; the same keys inside ``credentials`` are the legacy
+    # pre-column REST shape and are lifted into these columns on write (an
+    # explicit top-level value wins) and stripped from the stored blob.
+    token_exchange_endpoint: Optional[str] = None
+    audience: Optional[str] = None
+    subject_token_type: Optional[str] = None
+    token_exchange_profile: Optional[str] = None
     allow_all_keys: bool = False
     available_on_public_internet: bool = True
     delegate_auth_to_upstream: bool = False
@@ -2322,6 +2340,28 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
             "is active as a reminder that hard enforcement is relaxed."
         ),
     )
+    user_url_validation: Optional[bool] = Field(
+        None,
+        description=(
+            "Master switch for the SSRF guard applied to user-supplied URLs "
+            "(image_url, file_url, MCP/OpenAPI spec URLs, etc). Defaults to True. "
+            "Set to False to disable DNS/IP validation entirely (not recommended)."
+        ),
+    )
+    user_url_allowed_hosts: Optional[list[str]] = Field(
+        None,
+        description=(
+            "SSRF allowlist for user-supplied URLs. Entries are `hostname` or "
+            "`hostname:port` (bracketed for IPv6, e.g. `[::1]:8080`). Allowlisted "
+            "hosts skip the blocked-network check in validate_url() but still "
+            "resolve DNS. Use this to permit legitimate internal targets, e.g. "
+            "an internal OpenAPI/MCP server."
+        ),
+    )
+    provider_url_destination_allowed_hosts: Optional[list[str]] = Field(
+        None,
+        description="Allowlist of hosts a request may redirect a provider call's destination URL to.",
+    )
 
 
 class ConfigYAML(LiteLLMPydanticObjectBase):
@@ -2447,6 +2487,7 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView):  # the expected response ob
     request_route: Optional[str] = None
     is_session_token: bool = False
     budget_reservation: Optional[Dict[str, Any]] = Field(default=None, exclude=True)
+    budget_throttle_pct: Optional[float] = Field(default=None, exclude=True)
     user: Optional[Any] = None  # Expanded user object when expand=user is used
     created_by_user: Optional[Any] = None  # Expanded created_by user when expand=user is used
     end_user_object_permission: Optional[LiteLLM_ObjectPermissionTable] = None
@@ -3829,6 +3870,7 @@ LiteLLM_ManagementEndpoint_MetadataFields = [
     "model_rpm_limit",
     "model_tpm_limit",
     "mcp_rpm_limit",
+    "tag_rpm_limit",
     "rpm_limit_type",
     "tpm_limit_type",
     "enforced_params",
@@ -3837,6 +3879,7 @@ LiteLLM_ManagementEndpoint_MetadataFields = [
     "allowed_vector_store_indexes",
     "enforced_batch_output_expires_after",
     "enforced_file_expires_after",
+    "throttle_on_budget_exceeded",
 ]
 
 LiteLLM_ManagementEndpoint_MetadataFields_Premium = [

litellm/proxy/auth/auth_checks.py

@@ -60,6 +60,10 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.route_checks import RouteChecks
+from litellm.proxy.auth.budget_throttle import (
+    budget_throttle_percentage,
+    should_throttle_budget_exceeded,
+)
 from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
 from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
 from litellm.proxy.common_utils.http_parsing_utils import (
@@ -1470,7 +1474,7 @@ def _should_check_db(key: str, last_db_access_time: LimitedSizeOrderedDict, db_c
     elif last_db_access_time[key][0] is not None:  # check db for non-null values (for refresh operations)
         return True
     elif last_db_access_time[key][0] is None:
-        if current_time - last_db_access_time[key] >= db_cache_expiry:
+        if current_time - last_db_access_time[key][1] >= db_cache_expiry:
             return True
     return False
 
@@ -1645,6 +1649,12 @@ async def get_user_object(
                     include={"organization_memberships": True},
                 )
             else:
+                if should_check_db:
+                    _update_last_db_access_time(
+                        key=db_access_time_key,
+                        value=None,
+                        last_db_access_time=last_db_access_time,
+                    )
                 raise Exception
 
         if response.organization_memberships is not None and len(response.organization_memberships) > 0:
@@ -2496,6 +2506,7 @@ def _copy_user_api_key_auth_for_cache(
 ) -> UserAPIKeyAuth:
     copied_key_obj = user_api_key_obj.model_copy()
     copied_key_obj.budget_reservation = None
+    copied_key_obj.budget_throttle_pct = None
     copied_key_obj.parent_otel_span = None
     copied_key_obj.request_route = None
     return copied_key_obj
@@ -3428,6 +3439,24 @@ async def is_valid_fallback_model(
     return True
 
 
+def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool:
+    """
+    Throttle an over-budget key instead of blocking it, when the key opted in
+    via `throttle_on_budget_exceeded` and a global percentage is configured.
+
+    Records the percentage on the request-scoped `budget_throttle_pct` so the
+    rate limiter scales the key's TPM/RPM down to it; the persistent limits are
+    left untouched so the throttle never compounds across requests. Returns True
+    when the key was throttled (caller skips raising), False when it should still
+    be hard-blocked.
+    """
+    pct = budget_throttle_percentage()
+    if pct is None or not should_throttle_budget_exceeded(valid_token):
+        return False
+    valid_token.budget_throttle_pct = pct
+    return True
+
+
 async def _virtual_key_max_budget_check(
     valid_token: UserAPIKeyAuth,
     proxy_logging_obj: ProxyLogging,
@@ -3488,6 +3517,8 @@ async def _virtual_key_max_budget_check(
         # so a NaN max_budget would silently disable enforcement.  Treat a
         # non-finite max_budget as "no configured limit" rather than as a bypass.
         if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget:
+            if _apply_budget_exceeded_throttle(valid_token):
+                return
             # name the key in the error so operators don't have to reverse-map
             # spend back to a key; key_name is the masked form (last 4 chars)
             key_label = valid_token.key_alias or "key"

litellm/proxy/auth/auth_utils.py

@@ -975,6 +975,20 @@ def get_team_mcp_rpm_limit(
     return None
 
 
+def get_key_tag_rpm_limit(
+    user_api_key_dict: UserAPIKeyAuth,
+) -> Optional[dict[str, int]]:
+    """
+    Get the per-request-tag rpm limit configured on a given api key.
+
+    The returned dict is keyed by request tag, so each tag/group tracked on
+    the key gets its own independent RPM counter.
+    """
+    if user_api_key_dict.metadata:
+        return user_api_key_dict.metadata.get("tag_rpm_limit")
+    return None
+
+
 def get_project_model_rpm_limit(
     user_api_key_dict: UserAPIKeyAuth,
 ) -> Optional[Dict[str, int]]:

litellm/proxy/auth/budget_throttle.py

@@ -0,0 +1,56 @@
+"""
+Throttle a key after it exceeds its own ``max_budget`` instead of blocking it.
+
+When a key opts in via ``throttle_on_budget_exceeded`` and a global
+``budget_exceeded_throttle_percentage`` is configured, an over-budget key keeps
+serving requests but at a reduced TPM/RPM (the configured percentage of its
+configured limits). The decision (over budget + opted in) is made once during
+auth; the scaling is recomputed from the key's original limits on every request
+so it never compounds across requests.
+"""
+
+import math
+from typing import Optional
+
+import litellm
+from litellm.proxy._types import UserAPIKeyAuth
+
+
+def budget_throttle_percentage() -> Optional[float]:
+    """
+    The global throttle percentage, or None when throttling is disabled /
+    misconfigured (in which case an over-budget key is hard-blocked, the safe
+    default).
+    """
+    pct = litellm.budget_exceeded_throttle_percentage
+    if not isinstance(pct, (int, float)) or isinstance(pct, bool):
+        return None
+    if not 0 < pct <= 1:
+        return None
+    return float(pct)
+
+
+def should_throttle_budget_exceeded(valid_token: UserAPIKeyAuth) -> bool:
+    """
+    True when a key that exceeded its own ``max_budget`` should be throttled
+    rather than blocked: it opted in, a valid global percentage is set, and the
+    key has a TPM or RPM limit to scale down. A key with neither limit has
+    nothing to throttle, so it stays hard-blocked (the safe default) rather than
+    serving unlimited requests past its budget.
+    """
+    if (valid_token.metadata or {}).get("throttle_on_budget_exceeded") is not True:
+        return False
+    if valid_token.tpm_limit is None and valid_token.rpm_limit is None:
+        return False
+    return budget_throttle_percentage() is not None
+
+
+def throttled_limit(limit: Optional[int], pct: Optional[float]) -> Optional[int]:
+    """
+    Scale a TPM/RPM limit to ``pct`` of its value, keeping a trickle of at least
+    1 so a throttled key is slowed rather than fully locked out. An unset limit
+    or unset percentage leaves the limit unchanged.
+    """
+    if limit is None or pct is None:
+        return limit
+    return max(1, math.floor(limit * pct))

litellm/proxy/auth/user_api_key_auth.py

@@ -2004,6 +2004,11 @@ async def _user_api_key_auth_builder(
 
             valid_token_dict = valid_token.model_dump(exclude_none=True)
             valid_token_dict.pop("token", None)
+            # budget_throttle_pct is excluded from model_dump (it must not leak
+            # into serialized responses), so carry the request-scoped decision
+            # forward by hand to the auth object the rate limiter receives.
+            if valid_token.budget_throttle_pct is not None:
+                valid_token_dict["budget_throttle_pct"] = valid_token.budget_throttle_pct
 
             if _end_user_object is not None:
                 valid_token_dict.update(end_user_params)

litellm/proxy/db/prisma_client.py

@@ -137,8 +137,17 @@ class PrismaWrapper:
         self.on_engine_replaced: Callable[[], None] | None = None
 
     def _get_engine_pid(self) -> int:
-        """Get the PID of the current Prisma engine subprocess, or 0 if unavailable."""
+        """Get the PID of the current Prisma engine subprocess, or 0 if unavailable.
+
+        Must never raise: it runs inside the reconnect path, where the client
+        may be in any broken state. Prisma's ``_engine`` is a property that
+        raises ``ClientNotConnectedError`` on a disconnected client; if that
+        escaped here, ``recreate_prisma_client`` would fail before it could
+        build a replacement client and the reconnect loop could never recover.
+        """
         try:
+            if self._original_prisma.is_connected() is not True:
+                return 0
             engine = self._original_prisma._engine
             process = getattr(engine, "process", None) if engine is not None else None
             if process is not None:

litellm/proxy/guardrails/_content_utils.py

@@ -32,6 +32,9 @@ def is_text_content_call_type(call_type: str) -> bool:
     return call_type in TEXT_CONTENT_CALL_TYPES
 
 
+TEXT_PART_TYPES: FrozenSet[str] = frozenset({"text", "input_text", "output_text"})
+
+
 def _iter_text_parts_in_content(content: Any) -> Iterator[str]:
     """Yield text fragments from a ``message.content`` value (string or
     multimodal list). Non-text parts (images, audio, …) are skipped."""
@@ -48,7 +51,7 @@ def _iter_text_parts_in_content(content: Any) -> Iterator[str]:
                 continue
             if not isinstance(part, dict):
                 continue
-            if part.get("type") == "text":
+            if part.get("type") in TEXT_PART_TYPES:
                 text = part.get("text")
                 if isinstance(text, str) and text:
                     yield text
@@ -58,14 +61,20 @@ def _coerce_input_to_messages(input_value: Any) -> List[Dict[str, Any]]:
     """Coerce a Responses-API ``data["input"]`` value into chat-style messages."""
     if isinstance(input_value, str):
         return [{"role": "user", "content": input_value}]
-    if isinstance(input_value, list):
-        if input_value and all(isinstance(item, dict) and "role" in item for item in input_value):
-            return list(input_value)
-        # Mixed lists (content-part dicts + bare strings) and pure
-        # string/dict lists all become a single user message; the content
-        # iterator below handles each element type uniformly.
-        return [{"role": "user", "content": input_value}]
-    return []
+    if not isinstance(input_value, list):
+        return []
+    messages: List[Dict[str, Any]] = []
+    for item in input_value:
+        if isinstance(item, str):
+            messages.append({"role": "user", "content": item})
+        elif isinstance(item, dict):
+            if item.get("type") in TEXT_PART_TYPES:
+                messages.append({"role": item.get("role") or "user", "content": [item]})
+            elif "content" in item:
+                messages.append({"role": item.get("role") or "user", "content": item["content"]})
+            elif item.get("type") == "function_call_output" and "output" in item:
+                messages.append({"role": item.get("role") or "tool", "content": item["output"]})
+    return messages
 
 
 def _iter_inspection_messages(data: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
@@ -112,7 +121,7 @@ def walk_user_text(data: Dict[str, Any], visit: Callable[[str], str]) -> int:
                     new_parts.append(visit(part))
                 elif (
                     isinstance(part, dict)
-                    and part.get("type") == "text"
+                    and part.get("type") in TEXT_PART_TYPES
                     and isinstance(part.get("text"), str)
                     and part["text"]
                 ):
@@ -136,25 +145,20 @@ def walk_user_text(data: Dict[str, Any], visit: Callable[[str], str]) -> int:
             data["input"] = visit(input_value)
         return visited
     if isinstance(input_value, list):
-        # List of full messages: rewrite each message's content.
-        if input_value and all(isinstance(item, dict) and "role" in item for item in input_value):
-            for item in input_value:
-                if "content" in item:
-                    item["content"] = _rewrite_content(item["content"])
-            return visited
-        # List of content parts and/or bare strings: rewrite in place.
         for idx, item in enumerate(input_value):
-            if isinstance(item, str) and item:
-                visited += 1
-                input_value[idx] = visit(item)
-            elif (
-                isinstance(item, dict)
-                and item.get("type") == "text"
-                and isinstance(item.get("text"), str)
-                and item["text"]
-            ):
-                visited += 1
-                input_value[idx] = {**item, "text": visit(item["text"])}
+            if isinstance(item, str):
+                if item:
+                    visited += 1
+                    input_value[idx] = visit(item)
+            elif isinstance(item, dict):
+                if item.get("type") in TEXT_PART_TYPES:
+                    if isinstance(item.get("text"), str) and item["text"]:
+                        visited += 1
+                        input_value[idx] = {**item, "text": visit(item["text"])}
+                elif "content" in item:
+                    item["content"] = _rewrite_content(item["content"])
+                elif item.get("type") == "function_call_output" and "output" in item:
+                    item["output"] = _rewrite_content(item["output"])
         return visited
 
     return visited

litellm/proxy/guardrails/guardrail_hooks/aim/aim.py

@@ -93,11 +93,10 @@ class AimGuardrail(CustomGuardrail):
             user_email=user_email,
             litellm_call_id=call_id,
         )
-        # Covers multimodal list content + Responses-API input.
         response = await self.async_handler.post(
             f"{self.api_base}/fw/v1/analyze",
             headers=headers,
-            json={"messages": build_inspection_messages(data)},
+            json={"messages": self._build_aim_inspection_messages(data)},
         )
         response.raise_for_status()
         res = response.json()
@@ -116,6 +115,15 @@ class AimGuardrail(CustomGuardrail):
             verbose_proxy_logger.error(f"Aim: {action_type} action")
         return data
 
+    @staticmethod
+    def _build_aim_inspection_messages(data: dict) -> list[dict[str, str]]:
+        """AIM validates against the OpenAI chat schema. Bare ``role: "tool"``
+        without ``tool_call_id`` and bare ``role: "function"`` without ``name``
+        are rejected; the flatten drops those fields, so any role outside
+        ``{system, user, assistant}`` collapses to ``user`` for the AIM POST."""
+        safe_roles = {"system", "user", "assistant"}
+        return [{**m, "role": "user"} if m["role"] not in safe_roles else m for m in build_inspection_messages(data)]
+
     @staticmethod
     def _rejection(message: str, *, openai_code: str | None = None) -> ProxyException:
         return ProxyException(
@@ -177,7 +185,10 @@ class AimGuardrail(CustomGuardrail):
                 user_email=user_email,
                 litellm_call_id=call_id,
             ),
-            json={"messages": build_inspection_messages(request_data) + [{"role": "assistant", "content": output}]},
+            json={
+                "messages": self._build_aim_inspection_messages(request_data)
+                + [{"role": "assistant", "content": output}]
+            },
         )
         response.raise_for_status()
         res = response.json()

litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py

@@ -213,7 +213,7 @@ class GraySwanGuardrail(CustomGuardrail):
             verbose_proxy_logger.debug("Gray Swan Guardrail: dynamic extra_body=%s", safe_dumps(dynamic_body))
 
         # Prepare and send payload
-        payload = self._prepare_payload(messages, dynamic_body, request_data)
+        payload = self._prepare_payload(messages, dynamic_body, request_data, logging_obj)
         if payload is None:
             return inputs
 
@@ -502,10 +502,38 @@ class GraySwanGuardrail(CustomGuardrail):
             "grayswan-api-key": self.api_key,
         }
 
+    def _extract_inbound_headers(
+        self,
+        request_data: dict,
+        logging_obj: Optional["LiteLLMLoggingObj"] = None,
+    ) -> Optional[dict[str, str]]:
+        headers = (request_data.get("proxy_server_request") or {}).get("headers")
+        if not headers:
+            headers = request_data.get("headers")
+        if not headers:
+            headers = (request_data.get("metadata") or {}).get("headers")
+        if not headers and logging_obj and getattr(logging_obj, "model_call_details", None):
+            headers = (
+                (logging_obj.model_call_details or {}).get("litellm_params", {}).get("metadata", {}).get("headers")
+            )
+        if not isinstance(headers, dict):
+            return None
+
+        forwarded_header_names = ("shade_scan_id",)
+        forwarded_headers = {}
+        for key, value in headers.items():
+            if str(key).lower() in forwarded_header_names:
+                forwarded_headers[str(key)] = str(value)
+        return forwarded_headers or None
+
     def _prepare_payload(
-        self, messages: List[Dict[str, str]], dynamic_body: dict, request_data: dict
-    ) -> Optional[Dict[str, Any]]:
-        payload: Dict[str, Any] = {"messages": messages}
+        self,
+        messages: list[dict[str, str]],
+        dynamic_body: dict,
+        request_data: dict,
+        logging_obj: Optional["LiteLLMLoggingObj"] = None,
+    ) -> Optional[dict[str, Any]]:
+        payload: dict[str, Any] = {"messages": messages}
 
         categories = dynamic_body.get("categories") or self.categories
         if categories:
@@ -523,10 +551,16 @@ class GraySwanGuardrail(CustomGuardrail):
         if "metadata" in dynamic_body:
             payload["metadata"] = dynamic_body["metadata"]
 
+        inbound_headers = self._extract_inbound_headers(request_data, logging_obj)
+
         litellm_metadata = request_data.get("litellm_metadata")
-        if isinstance(litellm_metadata, dict) and litellm_metadata:
-            cleaned_litellm_metadata = dict(litellm_metadata)
-            # cleaned_litellm_metadata.pop("user_api_key_auth", None)
+        cleaned_litellm_metadata = dict(litellm_metadata) if isinstance(litellm_metadata, dict) else {}
+        if inbound_headers:
+            existing_headers = cleaned_litellm_metadata.get("headers")
+            cleaned_litellm_metadata["headers"] = (
+                {**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers
+            )
+        if cleaned_litellm_metadata:
             sanitized = safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={})
             if isinstance(sanitized, dict) and sanitized:
                 payload["litellm_metadata"] = sanitized

litellm/proxy/hooks/mcp_semantic_filter/hook.py

@@ -129,6 +129,27 @@ class SemanticToolFilterHook(CustomLogger):
 
         return openai_tools_as_dicts
 
+    async def _filter_expanded_tools(
+        self,
+        data: dict,
+        expanded_tools: list[dict[str, Any]],
+    ) -> list[dict[str, Any]]:
+        """
+        Apply the semantic filter to expanded MCP tool definitions.
+
+        Expanded tools are flat OpenAI function dicts with a top-level
+        "name" (see transform_mcp_tool_to_openai_responses_api_tool), so
+        filter_tools can name-match them against the semantic router.
+        """
+        raw_messages = data.get("messages") or data.get("input") or []
+        messages = [{"role": "user", "content": raw_messages}] if isinstance(raw_messages, str) else raw_messages
+        user_query = self.filter.extract_user_query(messages)
+        if not user_query:
+            verbose_proxy_logger.debug("No user query found, skipping semantic filter on expanded MCP tools")
+            return expanded_tools
+
+        return await self.filter.filter_tools(query=user_query, available_tools=expanded_tools)
+
     def _is_mcp_tool(self, tool: object) -> bool:
         """
         Check whether *tool* is registered in the MCP semantic router.
@@ -184,6 +205,32 @@ class SemanticToolFilterHook(CustomLogger):
                 f"Semantic tool filter: all {len(native_tools)} tools are native, no MCP filtering applied"
             )
 
+    def _emit_filter_metadata_safe(
+        self,
+        data: dict,
+        mcp_tools: list[object],
+        filtered_mcp_tools: list[object],
+        native_tools: list[object],
+        filtered_tools: list[object],
+    ) -> None:
+        """
+        Emit filter metadata without letting an emission failure abort the
+        already-filtered request.
+        """
+        try:
+            self._emit_filter_metadata(
+                data=data,
+                mcp_tools=mcp_tools,
+                filtered_mcp_tools=filtered_mcp_tools,
+                native_tools=native_tools,
+                filtered_tools=filtered_tools,
+            )
+        except Exception as e:
+            verbose_proxy_logger.warning(
+                f"Failed to emit semantic filter metadata: {e}",
+                exc_info=True,
+            )
+
     async def async_pre_call_hook(
         self,
         user_api_key_dict: "UserAPIKeyAuth",
@@ -206,9 +253,6 @@ class SemanticToolFilterHook(CustomLogger):
             verbose_proxy_logger.debug("No tools in request, skipping semantic filter")
             return None
 
-        # Expanded MCP tools are in OpenAI nested format which
-        # filter_tools/_extract_tool_info cannot name-match, so we skip
-        # semantic filtering and return early.
         if self._should_expand_mcp_tools(tools):
             verbose_proxy_logger.debug("Detected litellm_proxy MCP references, expanding before semantic filtering")
 
@@ -227,11 +271,26 @@ class SemanticToolFilterHook(CustomLogger):
                     verbose_proxy_logger.warning("No tools expanded from MCP references")
                     return None
 
-                data["tools"] = native_tools_before_expand + expanded_tools
+                if not self.filter.enabled:
+                    data["tools"] = native_tools_before_expand + expanded_tools
+                    verbose_proxy_logger.debug("Semantic filter disabled, forwarding expanded MCP tools unfiltered")
+                    return data
+
+                filtered_expanded_tools = await self._filter_expanded_tools(data=data, expanded_tools=expanded_tools)
+
+                combined_tools = native_tools_before_expand + filtered_expanded_tools
+                data["tools"] = combined_tools
+                self._emit_filter_metadata_safe(
+                    data=data,
+                    mcp_tools=expanded_tools,
+                    filtered_mcp_tools=filtered_expanded_tools,
+                    native_tools=native_tools_before_expand,
+                    filtered_tools=combined_tools,
+                )
                 verbose_proxy_logger.info(
                     f"Expanded MCP references to {len(expanded_tools)} tools "
                     f"({len(native_tools_before_expand)} native preserved), "
-                    f"skipping semantic filter (OpenAI nested format)"
+                    f"semantic filter selected {len(filtered_expanded_tools)}"
                 )
                 return data
 
@@ -297,19 +356,13 @@ class SemanticToolFilterHook(CustomLogger):
 
             data["tools"] = filtered_tools
 
-            try:
-                self._emit_filter_metadata(
-                    data=data,
-                    mcp_tools=mcp_tools,
-                    filtered_mcp_tools=filtered_mcp_tools,
-                    native_tools=native_tools,
-                    filtered_tools=filtered_tools,
-                )
-            except Exception as e:
-                verbose_proxy_logger.warning(
-                    f"Failed to emit semantic filter metadata: {e}",
-                    exc_info=True,
-                )
+            self._emit_filter_metadata_safe(
+                data=data,
+                mcp_tools=mcp_tools,
+                filtered_mcp_tools=filtered_mcp_tools,
+                native_tools=native_tools,
+                filtered_tools=filtered_tools,
+            )
 
             return data
 

litellm/proxy/hooks/parallel_request_limiter.py

@@ -17,6 +17,7 @@ from litellm.proxy.auth.auth_utils import (
     get_key_model_rpm_limit,
     get_key_model_tpm_limit,
 )
+from litellm.proxy.auth.budget_throttle import throttled_limit
 from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
 from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
 
@@ -248,10 +249,11 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
         if data is None:
             data = {}
         global_max_parallel_requests = data.get("metadata", {}).get("global_max_parallel_requests", None)
-        tpm_limit = getattr(user_api_key_dict, "tpm_limit", sys.maxsize)
+        throttle_pct = getattr(user_api_key_dict, "budget_throttle_pct", None)
+        tpm_limit = throttled_limit(getattr(user_api_key_dict, "tpm_limit", sys.maxsize), throttle_pct)
         if tpm_limit is None:
             tpm_limit = sys.maxsize
-        rpm_limit = getattr(user_api_key_dict, "rpm_limit", sys.maxsize)
+        rpm_limit = throttled_limit(getattr(user_api_key_dict, "rpm_limit", sys.maxsize), throttle_pct)
         if rpm_limit is None:
             rpm_limit = sys.maxsize
 

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -31,7 +31,12 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_str_from_messages,
 )
 from litellm.proxy._types import UserAPIKeyAuth
-from litellm.proxy.auth.auth_utils import get_model_rate_limit_from_metadata
+from litellm.proxy.auth.auth_utils import (
+    get_key_tag_rpm_limit,
+    get_model_rate_limit_from_metadata,
+)
+from litellm.proxy.auth.budget_throttle import throttled_limit
+from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body
 from litellm.proxy.common_utils.proxy_rate_limit_error import (
     ProxyRateLimitError,
     map_v3_rate_limit_type,
@@ -1299,6 +1304,43 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             )
         )
 
+    def _add_tag_per_key_rate_limit_descriptor(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        data: dict,
+        descriptors: list[RateLimitDescriptor],
+    ) -> None:
+        """
+        Add per-request-tag rpm limit descriptors for the API key.
+
+        Each tag carried on the request that has a configured limit gets its own
+        ``{api_key}:{tag}`` counter, so a burst on one tag/group never consumes
+        another's budget. Tags without a configured limit fall through to the
+        key-level descriptor.
+        """
+        if not user_api_key_dict.api_key:
+            return
+
+        tag_rpm_limit = get_key_tag_rpm_limit(user_api_key_dict) or {}
+        if not tag_rpm_limit:
+            return
+
+        for tag in dict.fromkeys(get_tags_from_request_body(data)):
+            rpm_limit = tag_rpm_limit.get(tag)
+            if rpm_limit is None:
+                continue
+            descriptors.append(
+                RateLimitDescriptor(
+                    key="tag_per_key",
+                    value=f"{user_api_key_dict.api_key}:{tag}",
+                    rate_limit={
+                        "requests_per_unit": rpm_limit,
+                        "tokens_per_unit": None,
+                        "window_size": self.window_size,
+                    },
+                )
+            )
+
     def _add_mcp_per_key_rate_limit_descriptor(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -1549,18 +1591,19 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             or user_api_key_dict.tpm_limit is not None
             or user_api_key_dict.max_parallel_requests is not None
         ):
+            throttle_pct = user_api_key_dict.budget_throttle_pct
             descriptors.append(
                 RateLimitDescriptor(
                     key="api_key",
                     value=user_api_key_dict.api_key,
                     rate_limit={
                         "requests_per_unit": self._get_enforced_limit(
-                            limit_value=user_api_key_dict.rpm_limit,
+                            limit_value=throttled_limit(user_api_key_dict.rpm_limit, throttle_pct),
                             limit_type=rpm_limit_type,
                             model_has_failures=model_has_failures,
                         ),
                         "tokens_per_unit": self._get_enforced_limit(
-                            limit_value=user_api_key_dict.tpm_limit,
+                            limit_value=throttled_limit(user_api_key_dict.tpm_limit, throttle_pct),
                             limit_type=tpm_limit_type,
                             model_has_failures=model_has_failures,
                         ),
@@ -1643,6 +1686,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             descriptors=descriptors,
         )
 
+        # Per-request-tag rate limits scoped to this key
+        self._add_tag_per_key_rate_limit_descriptor(
+            user_api_key_dict=user_api_key_dict,
+            data=data,
+            descriptors=descriptors,
+        )
+
         # REST MCP calls pass the raw body through this hook before server
         # resolution; only the later synthetic hook payload may carry this key.
         if call_type == CallTypes.call_mcp_tool.value and "server_id" not in data:
@@ -1959,6 +2009,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
         # Org Level Rate Limits
         descriptors.extend(self.create_organization_rate_limit_descriptor(user_api_key_dict, requested_model))
+
         # Only check rate limits if we have descriptors with actual limits
         if descriptors:
             # First pass: RPM and max_parallel_requests sliding-window check.

litellm/proxy/management_endpoints/cache_settings_endpoints.py

@@ -38,13 +38,35 @@ from litellm.types.management_endpoints import (
 router = APIRouter()
 
 # Cache fields holding credentials. Masked on read so plaintext Redis /
-# Sentinel passwords never leave the server in a GET response.
-_CACHE_SENSITIVE_FIELDS: set = {"password", "sentinel_password"}
+# Sentinel passwords never leave the server in a GET response. `url` is here
+# because a Redis/Valkey URL can embed a password inline
+# (e.g. redis://:secret@host:6379/1).
+_CACHE_SENSITIVE_FIELDS: set = {"password", "sentinel_password", "url"}
 
 
 _REDACTED_VALUE = "***REDACTED***"
 
 
+_URL_OVERRIDDEN_CONNECTION_FIELDS: frozenset = frozenset({"host", "port", "db", "password", "username"})
+
+
+def _resolve_cache_url_precedence(settings: Mapping[str, Any]) -> dict[str, Any]:
+    """Return cache settings with the url-vs-discrete-fields ambiguity resolved.
+
+    When a full ``url`` is supplied it wins: the discrete
+    host/port/db/password/username fields are dropped so the persisted config
+    is unambiguous and matches runtime resolution in ``litellm._redis``
+    (``redis.Redis.from_url`` ignores them). Cluster mode
+    (``redis_startup_nodes``) is exempt because it authenticates via the
+    discrete fields rather than a url.
+    """
+    url = settings.get("url")
+    has_url = isinstance(url, str) and url.strip() != ""
+    if not has_url or settings.get("redis_startup_nodes"):
+        return dict(settings)
+    return {k: v for k, v in settings.items() if k not in _URL_OVERRIDDEN_CONNECTION_FIELDS}
+
+
 def _redact_settings(settings: Optional[Mapping[str, Any]]) -> Dict[str, Any]:
     """Replace every value in a settings map with a fixed marker.
 
@@ -311,7 +333,7 @@ async def test_cache_connection(
     from litellm import Cache
 
     try:
-        cache_settings = request.cache_settings.copy()
+        cache_settings = _resolve_cache_url_precedence(request.cache_settings)
         verbose_proxy_logger.debug("Testing cache connection with settings: %s", cache_settings)
 
         # Only support Redis for now
@@ -378,7 +400,7 @@ async def update_cache_settings(
         )
 
     try:
-        cache_settings = request.cache_settings.copy()
+        cache_settings = _resolve_cache_url_precedence(request.cache_settings)
 
         # Snapshot the prior settings (key set only — values get redacted in
         # the audit row) so the audit-log entry shows which fields changed.

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -377,6 +377,7 @@ async def new_user(
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
     - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user.
+    - tag_rpm_limit: Optional[dict] - Per-request-tag rpm limit, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Enforced for keys only; values set on a user are stored but not enforced per user.
     - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
     - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo").
     - agent_id: Optional[str] - The agent id associated with the user.
@@ -1379,6 +1380,7 @@ async def user_update(
         - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
         - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
         - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user.
+        - tag_rpm_limit: Optional[dict] - Per-request-tag rpm limit, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Enforced for keys only; values set on a user are stored but not enforced per user.
         - model_tpm_limit: Optional[float] - Model-specific tpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
         - spend: Optional[float] - Amount spent by user. Default is 0. Will be updated by proxy whenever user is used. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d"), months ("1mo").
         - agent_id: Optional[str] - The agent id associated with the user.

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -111,6 +111,7 @@ from litellm.repositories.verification_token_repository import (
     VerificationTokenRepository,
 )
 from litellm.router import Router
+from litellm.secret_managers.base_secret_manager import raise_if_unsafe_secret_name
 from litellm.secret_managers.main import get_secret
 from litellm.types.proxy.management_endpoints.key_management_endpoints import (
     BulkUpdateKeyRequest,
@@ -758,6 +759,12 @@ async def _common_key_generation_helper(
         premium_user=premium_user,
     )
 
+    if data.throttle_on_budget_exceeded is True and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value:
+        raise HTTPException(
+            status_code=403,
+            detail={"error": "Only proxy admins can enable throttle_on_budget_exceeded on a key."},
+        )
+
     if data.metadata is not None and data.metadata.get("service_account_id") is not None and data.team_id is None:
         await validate_team_id_used_in_service_account_request(
             team_id=data.team_id,
@@ -1483,12 +1490,14 @@ async def generate_key_fn(
     - guardrails: Optional[List[str]] - List of active guardrails for the key
     - policies: Optional[List[str]] - List of policy names to apply to the key. Policies define guardrails, conditions, and inheritance rules.
     - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key.
+    - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely.
     - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
     - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
     - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
     - mcp_rpm_limit: Optional[dict] - key-specific per-MCP-server rpm limit, keyed by MCP server name (alias if set, else the configured name). Example - {"github": 100, "slack": 200}. IF null or {} then no MCP-specific rpm limit.
+    - tag_rpm_limit: Optional[dict] - key-specific per-request-tag rpm limit, keyed by request tag. Example - {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; requests whose tag is absent fall back to the key-level rpm limit.
     - tpm_limit_type: Optional[str] - Type of tpm limit. Options: "best_effort_throughput" (no error if we're overallocating tpm), "guaranteed_throughput" (raise an error if we're overallocating tpm), "dynamic" (dynamically exceed limit when no 429 errors). Defaults to "best_effort_throughput".
     - rpm_limit_type: Optional[str] - Type of rpm limit. Options: "best_effort_throughput" (no error if we're overallocating rpm), "guaranteed_throughput" (raise an error if we're overallocating rpm), "dynamic" (dynamically exceed limit when no 429 errors). Defaults to "best_effort_throughput".
     - allowed_cache_controls: Optional[list] - List of allowed cache control values. Example - ["no-cache", "no-store"]. See all values - https://docs.litellm.ai/docs/proxy/caching#turn-on--off-caching-per-request
@@ -2317,6 +2326,16 @@ async def _validate_update_key_data(
         or "budget_limits" in data.model_fields_set
     )
 
+    _existing_metadata = getattr(existing_key_row, "metadata", None)
+    _existing_throttle = (
+        _existing_metadata.get("throttle_on_budget_exceeded") if isinstance(_existing_metadata, dict) else None
+    )
+    if data.throttle_on_budget_exceeded is True and _existing_throttle is not True and not _is_proxy_admin:
+        raise HTTPException(
+            status_code=403,
+            detail={"error": "Only proxy admins can enable throttle_on_budget_exceeded on a key."},
+        )
+
     # Personal-key bypass: the caller both created the key AND still owns it
     # (user_id == caller).  Checking only created_by would let a demoted admin
     # who originally created a key for another user continue editing it without
@@ -2497,6 +2516,7 @@ async def update_key_fn(
     - rpm_limit: Optional[int] - Requests per minute limit
     - model_rpm_limit: Optional[dict] - Model-specific RPM limits {"gpt-4": 100, "claude-v1": 200}
     - mcp_rpm_limit: Optional[dict] - Per-MCP-server RPM limits, keyed by MCP server name {"github": 100, "slack": 200}
+    - tag_rpm_limit: Optional[dict] - Per-request-tag RPM limits, keyed by request tag {"cell-1": 1000, "cell-2": 500}. Each tag gets an independent counter; absent tags fall back to the key-level rpm limit.
     - model_tpm_limit: Optional[dict] - Model-specific TPM limits {"gpt-4": 100000, "claude-v1": 200000}
     - tpm_limit_type: Optional[str] - TPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic"
     - rpm_limit_type: Optional[str] - RPM rate limit type - "best_effort_throughput", "guaranteed_throughput", or "dynamic"
@@ -2507,6 +2527,7 @@ async def update_key_fn(
     - guardrails: Optional[List[str]] - List of active guardrails for the key
     - policies: Optional[List[str]] - List of policy names to apply to the key. Policies define guardrails, conditions, and inheritance rules.
     - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the key.
+    - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely.
     - prompts: Optional[List[str]] - List of prompts that the key is allowed to use.
     - blocked: Optional[bool] - Whether the key is blocked
     - aliases: Optional[dict] - Model aliases for the key - [Docs](https://litellm.vercel.app/docs/proxy/virtual_keys#model-aliases)
@@ -3533,6 +3554,7 @@ async def generate_key_helper_fn(
     model_rpm_limit: Optional[dict] = None,
     model_tpm_limit: Optional[dict] = None,
     mcp_rpm_limit: Optional[dict] = None,
+    tag_rpm_limit: Optional[dict] = None,
     guardrails: Optional[list] = None,
     policies: Optional[list] = None,
     prompts: Optional[list] = None,
@@ -3606,6 +3628,9 @@ async def generate_key_helper_fn(
     if mcp_rpm_limit is not None:
         metadata = metadata or {}
         metadata["mcp_rpm_limit"] = mcp_rpm_limit
+    if tag_rpm_limit is not None:
+        metadata = metadata or {}
+        metadata["tag_rpm_limit"] = tag_rpm_limit
     if guardrails is not None:
         metadata = metadata or {}
         metadata["guardrails"] = guardrails
@@ -6267,8 +6292,13 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None:
     """
     Validate the format of the key_alias.
 
-    Gated behind ``litellm.enable_key_alias_format_validation`` (default **False**).
-    When disabled, no validation is performed so existing workflows are not broken.
+    A baseline validation always runs, regardless of
+    ``litellm.enable_key_alias_format_validation``.
+
+    The remaining charset/length rules are gated behind
+    ``litellm.enable_key_alias_format_validation`` (default **False**). When disabled,
+    only the baseline validation above is performed, so existing workflows are not
+    broken.
 
     Rules (when enabled):
     - None is OK (no alias).
@@ -6276,10 +6306,20 @@ def _validate_key_alias_format(key_alias: Optional[str]) -> None:
     - start/end with alphanumeric
     - only allow a-zA-Z0-9_-/.@
     """
-    if not litellm.enable_key_alias_format_validation:
+    if key_alias is None:
         return
 
-    if key_alias is None:
+    try:
+        raise_if_unsafe_secret_name(key_alias)
+    except ValueError:
+        raise ProxyException(
+            message="Invalid key_alias",
+            type=ProxyErrorTypes.bad_request_error,
+            param="key_alias",
+            code=400,
+        )
+
+    if not litellm.enable_key_alias_format_validation:
         return
 
     if not _KEY_ALIAS_PATTERN.match(key_alias):

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -537,6 +537,10 @@ if MCP_AVAILABLE:
         sanitized.authorization_url = None
         sanitized.token_url = None
         sanitized.registration_url = None
+        sanitized.token_exchange_endpoint = None
+        sanitized.audience = None
+        sanitized.subject_token_type = None
+        sanitized.token_exchange_profile = None
         # Drop env vars entirely rather than only blanking global values: the
         # names alone (DB_PASSWORD, GITHUB_API_KEY, ...) leak what secrets the
         # admin configured. Non-admins get the per-user vars they must fill in
@@ -578,6 +582,10 @@ if MCP_AVAILABLE:
         sanitized.authorization_url = None
         sanitized.token_url = None
         sanitized.registration_url = None
+        sanitized.token_exchange_endpoint = None
+        sanitized.audience = None
+        sanitized.subject_token_type = None
+        sanitized.token_exchange_profile = None
 
         sanitized.health_check_error = None
         sanitized.last_health_check = None
@@ -1905,6 +1913,11 @@ if MCP_AVAILABLE:
             expires_in=payload.expires_in,
             scopes=payload.scopes,
         )
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415
+            global_mcp_server_manager,
+        )
+
+        await global_mcp_server_manager.invalidate_user_oauth_token_cache(user_id, server_id)
         # Read back the persisted record so the response reflects the stored
         # expires_at rather than recomputing it here (which could diverge by
         # milliseconds or if the storage logic ever adds a grace period).
@@ -1945,6 +1958,11 @@ if MCP_AVAILABLE:
                 await delete_user_credential(prisma_client, user_id, server_id)
             except RecordNotFoundError:
                 pass  # Already gone — treat as a successful delete
+            from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415
+                global_mcp_server_manager,
+            )
+
+            await global_mcp_server_manager.invalidate_user_oauth_token_cache(user_id, server_id)
         return MCPOAuthUserCredentialStatus(
             server_id=server_id,
             has_credential=False,

litellm/proxy/management_endpoints/team_endpoints.py

@@ -26,6 +26,7 @@ from litellm._uuid import uuid
 from litellm.integrations.prometheus import PrometheusLogger
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.proxy._types import (
+    UI_TEAM_ID,
     BlockTeamRequest,
     CommonProxyErrors,
     DeleteTeamRequest,
@@ -1046,6 +1047,13 @@ async def new_team(
         if data.team_id is None:
             data.team_id = str(uuid.uuid4())
         else:
+            if data.team_id == UI_TEAM_ID:
+                raise HTTPException(
+                    status_code=400,
+                    detail={
+                        "error": f"team_id '{UI_TEAM_ID}' is reserved for LiteLLM UI dashboard sessions and cannot be used for a real team. Please use a different team id."
+                    },
+                )
             # Check if team_id exists already
             _existing_team_id = await prisma_client.get_data(
                 team_id=data.team_id, table_name="team", query_type="find_unique"

litellm/proxy/management_endpoints/ui_sso.py

@@ -113,7 +113,7 @@ from litellm.proxy.utils import (
 from litellm.repositories.table_repositories import SSOConfigRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
-from litellm.secret_managers.main import get_secret_bool, str_to_bool
+from litellm.secret_managers.main import get_secret_bool, get_secret_str, str_to_bool
 from litellm.types.proxy.management_endpoints.ui_sso import *  # noqa: F403
 from litellm.types.proxy.management_endpoints.ui_sso import (
     DefaultTeamSSOParams,
@@ -2134,12 +2134,16 @@ async def cli_poll_key(
                 models=session_data.get("models", []),
             )
 
-            user_db_obj = await get_user_object(
-                user_id=user_id,
-                prisma_client=prisma_client,
-                user_api_key_cache=user_api_key_cache,
-                user_id_upsert=False,
-            )
+            try:
+                user_db_obj = await get_user_object(
+                    user_id=user_id,
+                    prisma_client=prisma_client,
+                    user_api_key_cache=user_api_key_cache,
+                    user_id_upsert=False,
+                )
+            except ValueError as e:
+                verbose_proxy_logger.debug(f"CLI poll: user lookup failed, proceeding without user budget: {e}")
+                user_db_obj = None
             user_budget = user_db_obj.max_budget if user_db_obj is not None else None
 
             team_budget: Optional[float] = None
@@ -3733,8 +3737,7 @@ class MicrosoftSSOHandler:
     Handles Microsoft SSO callback response and returns a CustomOpenID object
     """
 
-    graph_api_base_url = "https://graph.microsoft.com/v1.0"
-    graph_api_user_groups_endpoint = f"{graph_api_base_url}/me/memberOf"
+    DEFAULT_GRAPH_API_BASE_URL = "https://graph.microsoft.com/v1.0"
 
     """
     Constants
@@ -3744,6 +3747,19 @@ class MicrosoftSSOHandler:
     # used for debugging to show the user groups litellm found from Graph API
     GRAPH_API_RESPONSE_KEY = "graph_api_user_groups"
 
+    @staticmethod
+    def get_graph_api_base_url() -> str:
+        """
+        Returns the Microsoft Graph API base URL, configurable via the
+        `MICROSOFT_GRAPH_ENDPOINT` env var so non-default clouds such as Azure
+        Government (GCC High) can point at `https://graph.microsoft.us/v1.0`
+        """
+        return get_secret_str("MICROSOFT_GRAPH_ENDPOINT") or MicrosoftSSOHandler.DEFAULT_GRAPH_API_BASE_URL
+
+    @staticmethod
+    def get_graph_api_user_groups_endpoint() -> str:
+        return f"{MicrosoftSSOHandler.get_graph_api_base_url()}/me/memberOf"
+
     @staticmethod
     async def get_microsoft_callback_response(
         request: Request,
@@ -3920,7 +3936,7 @@ class MicrosoftSSOHandler:
 
             # Fetch user membership from Microsoft Graph API
             all_group_ids = []
-            next_link: Optional[str] = MicrosoftSSOHandler.graph_api_user_groups_endpoint
+            next_link: Optional[str] = MicrosoftSSOHandler.get_graph_api_user_groups_endpoint()
             auth_headers = {"Authorization": f"Bearer {access_token}"}
             page_count = 0
 
@@ -4003,7 +4019,7 @@ class MicrosoftSSOHandler:
 
         Users use Enterprise Applications to manage Groups and Users on Microsoft Entra ID
         """
-        base_url = "https://graph.microsoft.com/v1.0"
+        base_url = MicrosoftSSOHandler.get_graph_api_base_url()
         # Endpoint to get app role assignments for the given service principal
         endpoint = f"/servicePrincipals/{service_principal_id}/appRoleAssignedTo"
         url = base_url + endpoint

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -7,7 +7,7 @@ import traceback
 from base64 import b64encode
 from datetime import datetime
 from itertools import groupby
-from typing import Any, Dict, List, Mapping, Optional, Tuple, Union, cast
+from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Tuple, Union, cast
 from urllib.parse import urlencode, urlparse
 
 import httpx
@@ -389,18 +389,24 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         forward_multipart: bool = False,
     ) -> httpx.Response:
         """
-        Handle non-streaming HTTP requests
+        Handle non-SSE HTTP requests
 
-        Handles special cases when GET requests, multipart/form-data requests, and generic httpx requests
+        Handles special cases when GET requests, multipart/form-data requests, and generic httpx requests.
+
+        GET and generic requests are sent with httpx stream semantics so the caller can
+        decide from the response headers whether to buffer the body (JSON, inspected for
+        logging/guardrails) or relay it to the client without materializing it in memory
+        (LIT-4009: large batch results files must not be buffered in proxy RSS).
         """
         if request.method == "GET":
-            response = await async_client.request(
-                method=request.method,
-                url=url,
+            get_request = async_client.build_request(
+                request.method,
+                url,
                 headers=headers,
                 params=requested_query_params,
             )
-        elif HttpPassThroughEndpointHelpers.is_multipart(request) is True and forward_multipart:
+            return await async_client.send(get_request, stream=True)
+        if HttpPassThroughEndpointHelpers.is_multipart(request) is True and forward_multipart:
             # Forward multipart via make_multipart_http_request even when _parsed_body is
             # non-empty (pass_through_request always injects litellm_logging_obj, etc.).
             # forward_multipart is False when custom_body was supplied (JSON body despite
@@ -412,16 +418,14 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
                 headers=headers,
                 requested_query_params=requested_query_params,
             )
-        else:
-            # Generic httpx method
-            response = await async_client.request(
-                method=request.method,
-                url=url,
-                headers=headers,
-                params=requested_query_params,
-                json=_parsed_body,
-            )
-        return response
+        generic_request = async_client.build_request(
+            request.method,
+            url,
+            headers=headers,
+            params=requested_query_params,
+            json=_parsed_body,
+        )
+        return await async_client.send(generic_request, stream=True)
 
     @staticmethod
     def is_multipart(request: Request) -> bool:
@@ -820,21 +824,7 @@ async def pass_through_request(
             forward_headers=forward_headers,
         )
 
-        # Apply default query parameters if provided, regardless of merge_query_params setting
-        if default_query_params or merge_query_params:
-            # Determine what to merge based on settings
-            request_params = dict(request.query_params) if merge_query_params else {}
-
-            # Create a new URL with the merged query params
-            url = url.copy_with(
-                query=urlencode(
-                    HttpPassThroughEndpointHelpers.get_merged_query_parameters(
-                        existing_url=url,
-                        request_query_params=request_params,
-                        default_query_params=default_query_params,
-                    )
-                ).encode("ascii")
-            )
+        requested_query_params: Optional[dict] = query_params or dict(request.query_params)
 
         endpoint_type: EndpointType = HttpPassThroughEndpointHelpers.get_endpoint_type(str(url))
 
@@ -952,9 +942,6 @@ async def pass_through_request(
         )
         logging_obj.model_call_details["litellm_call_id"] = litellm_call_id
 
-        # combine url with query params for logging
-        requested_query_params: Optional[dict] = query_params or dict(request.query_params)
-
         ## PASSTHROUGH MANAGED ID RESOLUTION (INPUT) ##
         # Resolve managed IDs in path, query params, and body back to raw
         # provider IDs before forwarding upstream.  Gated by feature flag and
@@ -1024,6 +1011,20 @@ async def pass_through_request(
                     request.method,
                 )
 
+        # Apply default query parameters if provided, regardless of merge_query_params setting
+        if default_query_params or merge_query_params:
+            # Create a new URL with the merged query params
+            url = url.copy_with(
+                query=urlencode(
+                    HttpPassThroughEndpointHelpers.get_merged_query_parameters(
+                        existing_url=url,
+                        request_query_params=requested_query_params,
+                        default_query_params=default_query_params,
+                    )
+                ).encode("ascii")
+            )
+            requested_query_params = None
+
         ## PASSTHROUGH MANAGED LIST (DB-only response) ##
         # For GET /v1/files and GET /v1/batches passthrough routes, serve the
         # listing entirely from our DB so each caller only sees their own IDs.
@@ -1164,13 +1165,14 @@ async def pass_through_request(
         if state_raw_body is not None:
             # SigV4-signed callers (Bedrock) require the exact pre-signed bytes
             # to be forwarded so the signature/Content-Length stay valid.
-            response = await async_client.request(
-                method=request.method,
-                url=url,
+            raw_body_request = async_client.build_request(
+                request.method,
+                url,
                 headers=headers,
                 params=requested_query_params,
                 content=state_raw_body,
             )
+            response = await async_client.send(raw_body_request, stream=True)
         else:
             response = await HttpPassThroughEndpointHelpers.non_streaming_http_request_handler(
                 request=request,
@@ -1226,6 +1228,40 @@ async def pass_through_request(
                 status_code=response.status_code,
             )
 
+        if not _should_buffer_passthrough_response(response):
+            relay_custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
+                call_id=litellm_call_id,
+                model_id=None,
+                cache_key=None,
+                api_base=str(url._uri_reference),
+            )
+            relay_callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+                data=_parsed_body or {},
+                user_api_key_dict=user_api_key_dict,
+                response=response,
+                request_headers=dict(request.headers),
+            )
+            if relay_callback_headers:
+                relay_custom_headers.update(relay_callback_headers)
+
+            return StreamingResponse(
+                _relay_passthrough_response_bytes(
+                    response=response,
+                    request_body=_parsed_body or {},
+                    url_route=str(url),
+                    start_time=start_time,
+                    logging_obj=logging_obj,
+                    custom_llm_provider=custom_llm_provider,
+                    success_handler_kwargs=kwargs,
+                ),
+                status_code=response.status_code,
+                headers=HttpPassThroughEndpointHelpers.get_response_headers(
+                    headers=response.headers,
+                    custom_headers=relay_custom_headers,
+                ),
+            )
+
         content = await response.aread()
 
         ## POST-CALL GUARDRAILS ##
@@ -2214,6 +2250,70 @@ def _is_streaming_response(response: httpx.Response) -> bool:
     return False
 
 
+def _should_buffer_passthrough_response(response: httpx.Response) -> bool:
+    """
+    Decide from the response headers whether the body must be read into memory.
+
+    JSON bodies (and upstream errors) stay buffered: spend logging, guardrails and
+    managed-id rewriting inspect them, and they are small in practice. Everything
+    else (jsonl batch results, octet-stream files, ...) is relayed to the client
+    chunk by chunk so a large body is never resident in full (LIT-4009). A missing
+    content-type is buffered because the body cannot be classified.
+    """
+    if response.status_code >= 400:
+        return True
+    media_type = response.headers.get("content-type", "").split(";")[0].strip().lower()
+    return media_type in ("", "application/json") or media_type.endswith("+json")
+
+
+async def _relay_passthrough_response_bytes(
+    response: httpx.Response,
+    request_body: dict,
+    url_route: str,
+    start_time: datetime,
+    logging_obj: LiteLLMLoggingObj,
+    custom_llm_provider: Optional[str],
+    success_handler_kwargs: dict,
+) -> AsyncGenerator[bytes, None]:
+    """
+    Yield upstream bytes to the client without accumulating them, then fire the
+    passthrough success handler with response_body=None (uninspected body). The
+    finally block also runs on client disconnect (GeneratorExit) so partial
+    downloads still produce a spend-log row, mirroring chunk_processor; a
+    disconnect additionally logs a warning with the number of bytes relayed so
+    partial deliveries are distinguishable from complete ones in proxy logs.
+    """
+    bytes_relayed = 0
+    upstream_fully_relayed = False
+    try:
+        async for chunk in response.aiter_bytes():
+            bytes_relayed += len(chunk)
+            yield chunk
+        upstream_fully_relayed = True
+    finally:
+        if not upstream_fully_relayed:
+            verbose_proxy_logger.warning(
+                f"Passthrough stream for {url_route} ended before upstream body was fully relayed; "
+                f"{bytes_relayed} bytes were sent to the client"
+            )
+        await response.aclose()
+        GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+            async_coroutine=pass_through_endpoint_logging.pass_through_async_success_handler(
+                httpx_response=response,
+                response_body=None,
+                url_route=url_route,
+                result="",
+                start_time=start_time,
+                end_time=datetime.now(),
+                logging_obj=logging_obj,
+                cache_hit=False,
+                request_body=request_body,
+                custom_llm_provider=custom_llm_provider,
+                **success_handler_kwargs,
+            )
+        )
+
+
 def _extract_model_from_vertex_ai_setup(setup_response: dict) -> Optional[str]:
     """
     Extract the model name from Vertex AI Live setup response.

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -34,6 +34,18 @@ from .llm_provider_handlers.vertex_passthrough_logging_handler import (
 cohere_passthrough_logging_handler = CoherePassthroughLoggingHandler()
 
 
+def _safe_response_text(httpx_response: httpx.Response) -> str:
+    """
+    Streamed passthrough responses are relayed to the client without being read
+    into memory, so accessing .text on them raises ResponseNotRead. Their body is
+    intentionally uninspected; log an empty string instead of failing the row.
+    """
+    try:
+        return httpx_response.text
+    except httpx.ResponseNotRead:
+        return ""
+
+
 class PassThroughEndpointLogging:
     def __init__(self):
         self.TRACKED_VERTEX_ROUTES = [
@@ -306,7 +318,9 @@ class PassThroughEndpointLogging:
             ]
             kwargs = normalized_llm_passthrough_logging_payload["kwargs"]
         if standard_logging_response_object is None:
-            standard_logging_response_object = StandardPassThroughResponseObject(response=httpx_response.text)
+            standard_logging_response_object = StandardPassThroughResponseObject(
+                response=_safe_response_text(httpx_response)
+            )
 
         kwargs = self._set_cost_per_request(
             logging_obj=logging_obj,

litellm/proxy/proxy_server.py

@@ -1127,45 +1127,6 @@ _OPENAPI_HTTP_METHODS = {
 # `_SSO_SENSITIVE_FIELDS` / `_CACHE_SENSITIVE_FIELDS` constants in the SSO
 # and cache endpoint files.
 _ALERTING_SENSITIVE_VARS: Set[str] = {"SLACK_WEBHOOK_URL", "SMTP_PASSWORD"}
-_DB_LITELLM_PARAM_ENV_REF_KEYS = frozenset(
-    {
-        "api_key",
-        "client_secret",
-        "vertex_credentials",
-        "vertex_ai_credentials",
-        "aws_access_key_id",
-        "aws_secret_access_key",
-        "aws_session_token",
-        "aws_region_name",
-        "aws_session_name",
-        "aws_profile_name",
-        "aws_role_name",
-        "aws_web_identity_token",
-        "aws_sts_endpoint",
-        "aws_external_id",
-        "aws_bedrock_runtime_endpoint",
-        "aws_bedrock_project_id",
-        "aws_batch_role_arn",
-        "aws_workspace_id",
-    }
-)
-
-
-def _db_model_is_team_scoped(model: object) -> bool:
-    model_info = getattr(model, "model_info", None)
-    if isinstance(model_info, BaseModel):
-        return getattr(model_info, "team_id", None) is not None
-    if isinstance(model_info, str):
-        try:
-            model_info = json.loads(model_info)
-        except (TypeError, ValueError):
-            model_info = None
-    if isinstance(model_info, dict) and model_info.get("team_id") is not None:
-        return True
-    if getattr(model_info, "team_id", None) is not None:
-        return True
-    model_name = getattr(model, "model_name", None)
-    return isinstance(model_name, str) and model_name.startswith("model_name_")
 
 
 def _strip_operation_id_method_suffix(operation_id: str) -> str:
@@ -3596,6 +3557,28 @@ def _scrub_db_overlay_remote_module_loads(section: str, db_value: Any) -> Any:
     return sanitized
 
 
+def _normalize_user_url_validation(value: object) -> Optional[bool]:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return str_to_bool(value)
+    return bool(value)
+
+
+def _apply_ssrf_general_settings(settings: Mapping[str, object]) -> None:
+    if "user_url_allowed_hosts" in settings:
+        litellm.user_url_allowed_hosts = cast(list[str], settings["user_url_allowed_hosts"])
+
+    user_url_validation = _normalize_user_url_validation(settings.get("user_url_validation"))
+    if user_url_validation is not None:
+        litellm.user_url_validation = user_url_validation
+
+    if "provider_url_destination_allowed_hosts" in settings:
+        litellm.provider_url_destination_allowed_hosts = cast(
+            list[str], settings["provider_url_destination_allowed_hosts"]
+        )
+
+
 class ProxyConfig:
     """
     Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
@@ -4583,6 +4566,9 @@ class ProxyConfig:
                     RoleBasedPermissions(**role_permission) for role_permission in rbac_role_permissions
                 ]
 
+            ### SSRF URL VALIDATION SETTINGS ###
+            _apply_ssrf_general_settings(general_settings)
+
             ## check if user has set a premium feature in general_settings
             if general_settings.get("enforced_params") is not None and premium_user is not True:
                 raise ValueError("Trying to use `enforced_params`" + CommonProxyErrors.not_premium_user.value)
@@ -5014,17 +5000,12 @@ class ProxyConfig:
                     deleted_deployments += 1
         return deleted_deployments
 
-    def _resolve_db_litellm_param(self, key: str, value: object, resolve_env_refs: bool = True) -> object:
+    def _resolve_db_litellm_param(self, key: str, value: object) -> object:
         if not isinstance(value, str):
             return value
 
         decrypted_value = decrypt_value_helper(value=value, key=key, return_original_value=True)
-        if (
-            resolve_env_refs
-            and key in _DB_LITELLM_PARAM_ENV_REF_KEYS
-            and isinstance(decrypted_value, str)
-            and decrypted_value.startswith("os.environ/")
-        ):
+        if isinstance(decrypted_value, str) and decrypted_value.startswith("os.environ/"):
             return get_secret(decrypted_value)
         return decrypted_value
 
@@ -5045,13 +5026,10 @@ class ProxyConfig:
         ## ADD MODEL LOGIC
         for m in db_models:
             _litellm_params = m.litellm_params
-            resolve_env_refs = not _db_model_is_team_scoped(m)
             if isinstance(_litellm_params, dict):
                 # decrypt values
                 for k, v in _litellm_params.items():
-                    _litellm_params[k] = self._resolve_db_litellm_param(
-                        key=k, value=v, resolve_env_refs=resolve_env_refs
-                    )
+                    _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v)
                 _litellm_params = LiteLLM_Params(**_litellm_params)
 
             else:
@@ -5077,15 +5055,12 @@ class ProxyConfig:
         _model_list: list = []
         for m in new_models:
             _litellm_params = m.litellm_params
-            resolve_env_refs = not _db_model_is_team_scoped(m)
             if isinstance(_litellm_params, BaseModel):
                 _litellm_params = _litellm_params.model_dump()
             if isinstance(_litellm_params, dict):
                 # decrypt values
                 for k, v in _litellm_params.items():
-                    _litellm_params[k] = self._resolve_db_litellm_param(
-                        key=k, value=v, resolve_env_refs=resolve_env_refs
-                    )
+                    _litellm_params[k] = self._resolve_db_litellm_param(key=k, value=v)
                 _litellm_params = LiteLLM_Params(**_litellm_params)
             else:
                 verbose_proxy_logger.error(
@@ -5620,6 +5595,15 @@ class ProxyConfig:
             if old_value != new_value:
                 await self._reschedule_spend_log_cleanup_job()
 
+        for key in (
+            "user_url_allowed_hosts",
+            "user_url_validation",
+            "provider_url_destination_allowed_hosts",
+        ):
+            if key in _general_settings:
+                general_settings[key] = _general_settings[key]
+        _apply_ssrf_general_settings(_general_settings)
+
     def _update_config_fields(
         self,
         current_config: dict,
@@ -14281,6 +14265,9 @@ async def update_config_general_settings(
             detail={"error": CommonProxyErrors.not_allowed_access.value},
         )
 
+    if data.field_name in _GENERAL_SETTINGS_UI_LITELLM_FIELDS:
+        return await _persist_general_settings_ui_litellm_field(data.field_name, data.field_value, user_api_key_dict)
+
     if data.field_name not in ConfigGeneralSettings.model_fields:
         raise HTTPException(
             status_code=400,
@@ -14339,6 +14326,7 @@ async def update_config_general_settings(
 
     if data.field_name == "plugins":
         register_plugins_from_config(general_settings)
+    _apply_ssrf_general_settings(general_settings)
 
     return response
 
@@ -14545,6 +14533,55 @@ async def get_config_general_settings(
             )
 
 
+_GENERAL_SETTINGS_UI_LITELLM_FIELDS: dict[str, dict[str, str]] = {
+    "budget_exceeded_throttle_percentage": {
+        "type": "Float",
+        "description": (
+            "Fraction (0, 1] of a key's configured TPM/RPM that an over-budget key with "
+            "'Throttle on budget exceeded' enabled keeps serving at. Leave empty to hard-block "
+            "over-budget keys."
+        ),
+    },
+}
+
+
+def _validate_general_settings_ui_litellm_value(field_name: str, value: Any) -> Optional[float]:
+    if value is None or value == "":
+        return None
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or not (0 < float(value) <= 1):
+        raise HTTPException(
+            status_code=400,
+            detail={"error": f"{field_name} must be a number in (0, 1] or empty"},
+        )
+    return float(value)
+
+
+async def _persist_general_settings_ui_litellm_field(
+    field_name: str, value: Any, user_api_key_dict: UserAPIKeyAuth
+) -> dict:
+    validated = _validate_general_settings_ui_litellm_value(field_name, value)
+    config = await proxy_config.get_config()
+    before_value = config.get("litellm_settings", {}).get(field_name)
+    setattr(litellm, field_name, validated)
+    if "litellm_settings" not in config:
+        config["litellm_settings"] = {}
+    config["litellm_settings"][field_name] = validated
+    await proxy_config.save_config(new_config=config)
+    asyncio.create_task(create_config_audit_log(field_name, "updated", before_value, validated, user_api_key_dict))
+    return {"message": f"Field {field_name} updated", "status": "success"}
+
+
+async def _reset_general_settings_ui_litellm_field(field_name: str, user_api_key_dict: UserAPIKeyAuth) -> dict:
+    config = await proxy_config.get_config()
+    before_value = config.get("litellm_settings", {}).get(field_name)
+    setattr(litellm, field_name, None)
+    if "litellm_settings" in config:
+        config["litellm_settings"].pop(field_name, None)
+    await proxy_config.save_config(new_config=config)
+    asyncio.create_task(create_config_audit_log(field_name, "deleted", before_value, None, user_api_key_dict))
+    return {"message": f"Field {field_name} reset", "status": "success"}
+
+
 @router.get(
     "/config/list",
     tags=["config.yaml"],
@@ -14698,6 +14735,35 @@ async def get_config_list(
                 )
                 return_val.append(_response_obj)
 
+    db_litellm_settings_row = await ConfigRepository(prisma_client).table.find_first(
+        where={"param_name": "litellm_settings"}
+    )
+    db_litellm_settings: dict = (
+        dict(db_litellm_settings_row.param_value)
+        if db_litellm_settings_row is not None and db_litellm_settings_row.param_value is not None
+        else {}
+    )
+    for litellm_field_name, spec in _GENERAL_SETTINGS_UI_LITELLM_FIELDS.items():
+        current_value: Optional[float] = getattr(litellm, litellm_field_name, None)
+        stored_in_db_litellm: Optional[bool]
+        if litellm_field_name in db_litellm_settings:
+            stored_in_db_litellm = True
+        elif current_value is not None:
+            stored_in_db_litellm = False
+        else:
+            stored_in_db_litellm = None
+        return_val.append(
+            ConfigList(
+                field_name=litellm_field_name,
+                field_type=spec["type"],
+                field_description=spec["description"],
+                field_value=current_value,
+                stored_in_db=stored_in_db_litellm,
+                field_default_value=None,
+                nested_fields=None,
+            )
+        )
+
     return return_val
 
 
@@ -14738,6 +14804,9 @@ async def delete_config_general_settings(
             },
         )
 
+    if data.field_name in _GENERAL_SETTINGS_UI_LITELLM_FIELDS:
+        return await _reset_general_settings_ui_litellm_field(data.field_name, user_api_key_dict)
+
     if data.field_name not in ConfigGeneralSettings.model_fields:
         raise HTTPException(
             status_code=400,

litellm/proxy/schema.prisma

@@ -329,6 +329,12 @@ model LiteLLM_MCPServerTable {
   token_url          String?
   registration_url   String?
   oauth2_flow        String?
+  token_exchange_endpoint String?
+  // Named for the RFC 8693 "audience" token-exchange request parameter (that flow only).
+  // RFC 8707 resource indicators are a separate concept, named "resource" in the v2 egress types.
+  audience           String?
+  subject_token_type String?
+  token_exchange_profile String?
   allow_all_keys Boolean @default(false)
   available_on_public_internet Boolean @default(true)
   delegate_auth_to_upstream Boolean @default(false)

litellm/proxy/spend_tracking/budget_reservation.py

@@ -17,6 +17,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.auth_utils import get_model_from_request
+from litellm.proxy.auth.budget_throttle import should_throttle_budget_exceeded
 from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.router import Router
@@ -54,6 +55,61 @@ def get_reserved_counter_keys(budget_reservation: Optional[dict]) -> set:
     }
 
 
+def _key_reservation_should_release_for_throttle(counter_key: str, valid_token: Optional[UserAPIKeyAuth]) -> bool:
+    """
+    Whether an over-budget key's own ``max_budget`` reservation should be
+    released rather than blocked, because the key opted into throttling: the
+    rate limiter slows it instead. Only the key's own ``max_budget`` counter is
+    exempt; team/user/window counters still enforce normally, and under-budget
+    requests never reach this branch so their concurrent-overspend protection is
+    untouched.
+    """
+    if valid_token is None:
+        return False
+    return counter_key == f"spend:key:{valid_token.token}" and should_throttle_budget_exceeded(valid_token)
+
+
+async def _apply_over_budget_reservation_policy(
+    counter: _BudgetCounter,
+    valid_token: Optional[UserAPIKeyAuth],
+    entry: dict[str, Any],
+    applied_entries: list[dict[str, Any]],
+    reservation_cost: float,
+    current_spend: float,
+) -> float:
+    """
+    Decide what to do when a counter is over budget, and return the reservation
+    cost to carry into the next counter. Three outcomes: an over-budget key that
+    opted into throttling releases its own reservation (the rate limiter slows
+    it) and keeps the cost; a partially-remaining budget resizes the reservation
+    down to what is left; anything else hard-blocks by raising.
+    """
+    if _key_reservation_should_release_for_throttle(counter.counter_key, valid_token):
+        await _release_applied_entries_best_effort(entries=[entry], default_reserved_cost=reservation_cost)
+        applied_entries.remove(entry)
+        return reservation_cost
+
+    remaining_before_reservation = counter.max_budget - (current_spend - reservation_cost)
+    if remaining_before_reservation > 1e-12:
+        await _resize_applied_reservation(
+            entries=applied_entries,
+            current_reserved_cost=reservation_cost,
+            new_reserved_cost=remaining_before_reservation,
+        )
+        return remaining_before_reservation
+
+    raise litellm.BudgetExceededError(
+        current_cost=current_spend,
+        max_budget=counter.max_budget,
+        message=(
+            "Budget has been exceeded! "
+            f"{counter.entity_type}={counter.entity_id} "
+            f"Current cost: {current_spend}, "
+            f"Max budget: {counter.max_budget}"
+        ),
+    )
+
+
 async def reserve_budget_for_request(
     request_body: dict,
     route: str,
@@ -130,25 +186,15 @@ async def reserve_budget_for_request(
                     cached_spend = await _get_current_counter_value(counter=counter)
                 current_spend = cached_spend + reservation_cost
             if current_spend > counter.max_budget:
-                remaining_before_reservation = counter.max_budget - (current_spend - reservation_cost)
-                if remaining_before_reservation > 1e-12:
-                    await _resize_applied_reservation(
-                        entries=applied_entries,
-                        current_reserved_cost=reservation_cost,
-                        new_reserved_cost=remaining_before_reservation,
-                    )
-                    reservation_cost = remaining_before_reservation
-                    continue
-                raise litellm.BudgetExceededError(
-                    current_cost=current_spend,
-                    max_budget=counter.max_budget,
-                    message=(
-                        "Budget has been exceeded! "
-                        f"{counter.entity_type}={counter.entity_id} "
-                        f"Current cost: {current_spend}, "
-                        f"Max budget: {counter.max_budget}"
-                    ),
+                reservation_cost = await _apply_over_budget_reservation_policy(
+                    counter=counter,
+                    valid_token=valid_token,
+                    entry=entry,
+                    applied_entries=applied_entries,
+                    reservation_cost=reservation_cost,
+                    current_spend=current_spend,
                 )
+                continue
     except Exception:
         await _release_applied_entries_best_effort(
             entries=applied_entries,

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -45,6 +45,8 @@ else:
 
 router = APIRouter()
 
+SPEND_LOGS_PAGINATION_COUNT_CAP = 10000
+
 
 @router.get(
     "/spend/keys",
@@ -1958,6 +1960,20 @@ async def ui_view_spend_logs(
         else:
             _order_expr = order_column
 
+        count_query = f"""
+            SELECT COUNT(*) AS total_count
+            FROM (
+                SELECT 1
+                FROM "LiteLLM_SpendLogs"
+                WHERE {" AND ".join(sql_conditions)}
+                LIMIT ${p}
+            ) AS bounded_matches
+        """
+        count_rows = await prisma_client.db.query_raw(count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1)
+        raw_total = int(count_rows[0]["total_count"]) if count_rows else 0
+        total_is_capped = raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP
+        total_records = SPEND_LOGS_PAGINATION_COUNT_CAP if total_is_capped else raw_total
+
         sql_query = f"""
             SELECT
                 request_id, call_type, api_key, spend, total_tokens,
@@ -1967,8 +1983,7 @@ async def ui_view_spend_logs(
                 cache_hit, cache_key, request_tags, team_id,
                 organization_id, end_user, requester_ip_address,
                 session_id, status, mcp_namespaced_tool_name, agent_id,
-                COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms,
-                COUNT(*) OVER () AS total_count
+                COALESCE(request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER) AS request_duration_ms
             FROM "LiteLLM_SpendLogs"
             WHERE {" AND ".join(sql_conditions)}
             ORDER BY {_order_expr} {_sql_dir}{_nulls_clause}
@@ -1978,34 +1993,13 @@ async def ui_view_spend_logs(
 
         data = await prisma_client.db.query_raw(sql_query, *sql_params)
 
-        # `COUNT(*) OVER ()` folds the total-match count into the same scan as the
-        # page data; a standalone `COUNT(*)` is a distributed RPC on sharded
-        # engines like YugabyteDB that contacts every tablet and times out
-        # regardless of row count (LIT-4027). The hot path (page 1 and in-range
-        # pages) always carries the count on its rows, so the count round trip is
-        # gone there. Only an out-of-range page overshoots the last row and comes
-        # back empty; fall back to a direct count there so total/total_pages stay
-        # accurate rather than collapsing to zero.
-        if data:
-            total_records = int(data[0]["total_count"])
-        elif page > 1:
-            total_records = int(
-                await SpendLogsRepository(prisma_client).table.count(
-                    where=where_conditions,
-                )
-            )
-        else:
-            total_records = 0
-
         # query_raw returns the JSONB `metadata` column as a string (the Prisma
         # serialiser bypasses the model-layer JSON hydration we get on the ORM
         # path). The UI reads `metadata.status` / `metadata.error_information`
         # as object fields, so failure rows looked like successes (#29674).
-        # Re-hydrate to dict here. Also drop the window-function `total_count`
-        # helper column so it does not leak into the serialised rows.
+        # Re-hydrate to dict here.
         for row in data:
             if isinstance(row, dict):
-                row.pop("total_count", None)
                 md = row.get("metadata")
                 if isinstance(md, str):
                     try:
@@ -2026,6 +2020,7 @@ async def ui_view_spend_logs(
             page_size,
             total_pages,
             enrich_session_counts=not is_v2,
+            total_is_capped=total_is_capped,
         )
     except Exception as e:
         verbose_proxy_logger.exception(f"Error in ui_view_spend_logs: {e}")
@@ -3334,6 +3329,7 @@ async def _build_ui_spend_logs_response(
     page_size: int,
     total_pages: int,
     enrich_session_counts: bool = True,
+    total_is_capped: bool = False,
 ) -> dict:
     """
     Build the paginated response for the UI spend-logs endpoint.
@@ -3358,10 +3354,12 @@ async def _build_ui_spend_logs_response(
         total_pages: Total number of pages.
         enrich_session_counts: Whether to add ``session_total_count`` to each
             row.  Defaults to ``True``.
+        total_is_capped: Whether ``total_records`` was clamped to the
+            pagination count cap (there are more matching rows than the cap).
 
     Returns:
         A dict with ``data`` (enriched rows), ``total``, ``page``,
-        ``page_size``, and ``total_pages``.
+        ``page_size``, ``total_pages``, and ``total_is_capped``.
     """
     count_map: dict[str, int] = {}
     if enrich_session_counts:
@@ -3451,6 +3449,7 @@ async def _build_ui_spend_logs_response(
         "page": page,
         "page_size": page_size,
         "total_pages": total_pages,
+        "total_is_capped": total_is_capped,
     }
 
 

litellm/proxy/utils.py

@@ -4097,11 +4097,22 @@ class PrismaClient:
             raise e
 
     def _get_engine_pid(self) -> int:
+        """Get the PID of the writer's engine subprocess, or 0 if unavailable.
+
+        Must never raise: prisma's ``_engine`` property raises
+        ``ClientNotConnectedError`` on a disconnected client, and an exception
+        escaping from the reconnect path would leave it unable to recover.
+        """
         try:
-            engine = self.db._original_prisma._engine  # type: ignore[attr-defined]
+            prisma_obj = self.writer_db._original_prisma
+            if prisma_obj.is_connected() is not True:
+                return 0
+            engine = prisma_obj._engine
             process = getattr(engine, "process", None) if engine is not None else None
             if process is not None:
-                return process.pid
+                pid = process.pid
+                if isinstance(pid, int):
+                    return pid
         except (AttributeError, TypeError):
             pass
         return 0

litellm/realtime_api/main.py

@@ -515,6 +515,7 @@ async def _realtime_health_check(
     api_base: Optional[str] = None,
     api_version: Optional[str] = None,
     realtime_protocol: Optional[str] = None,
+    model_params: Optional[dict] = None,
 ):
     """
     Health check for realtime API - tries connection to the realtime API websocket
@@ -550,14 +551,17 @@ async def _realtime_health_check(
     elif custom_llm_provider == "xai":
         url = xai_realtime._construct_url(api_base=api_base or "https://api.x.ai/v1", query_params={"model": model})
     elif custom_llm_provider == "vertex_ai":
-        vertex_location = litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION")
-        resolved_location = vertex_llm_base.get_vertex_region(vertex_region=vertex_location, model=model)
+        vertex_model_params = model_params or {}
+        resolved_location = vertex_llm_base.get_vertex_region(
+            vertex_region=VertexBase.safe_get_vertex_ai_location(vertex_model_params),
+            model=model,
+        )
         (
             access_token,
             resolved_project,
         ) = await vertex_llm_base._ensure_access_token_async(
-            credentials=None,
-            project_id=litellm.vertex_project or get_secret_str("VERTEXAI_PROJECT"),
+            credentials=VertexBase.safe_get_vertex_ai_credentials(vertex_model_params),
+            project_id=VertexBase.safe_get_vertex_ai_project(vertex_model_params),
             custom_llm_provider="vertex_ai",
         )
         vertex_realtime_config = VertexAIRealtimeConfig(

litellm/rerank_api/main.py

@@ -158,7 +158,7 @@ def rerank(
             instruction=instruction,
             non_default_params=kwargs,
         )
-        verbose_logger.info(f"optional_rerank_params: {optional_rerank_params}")
+        verbose_logger.debug(f"optional_rerank_params: {optional_rerank_params}")
         if isinstance(optional_params.timeout, str):
             optional_params.timeout = float(optional_params.timeout)
 

litellm/responses/mcp/litellm_proxy_mcp_handler.py

@@ -16,8 +16,10 @@ from typing import (
 from litellm._logging import verbose_logger
 from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.proxy._experimental.mcp_server.utils import split_server_prefix_from_name
-from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
+from litellm.proxy._experimental.mcp_server.utils import (
+    split_server_prefix_from_name,
+    strip_known_server_prefix,
+)
 from litellm.responses.main import aresponses
 from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
 from litellm.types.llms.openai import ResponsesAPIResponse
@@ -628,6 +630,9 @@ class LiteLLM_Proxy_MCP_Handler:
         from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
             global_mcp_server_manager,
         )
+        from litellm.proxy._experimental.mcp_server.server import (
+            _resolve_display_name_to_original,
+        )
         from litellm.proxy.proxy_server import proxy_logging_obj
 
         tool_results = []
@@ -654,11 +659,13 @@ class LiteLLM_Proxy_MCP_Handler:
 
                 server_name = tool_server_map[tool_name]
 
-                # Remove the server name prefix if the tool name includes it.
-                sanitized_tool_name = tool_name
-                unprefixed_name, prefixed_server_name = split_server_prefix_from_name(tool_name)
-                if prefixed_server_name and prefixed_server_name == server_name and unprefixed_name:
-                    sanitized_tool_name = unprefixed_name
+                mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
+                    server_name
+                ) or global_mcp_server_manager._get_mcp_server_from_tool_name(tool_name)
+                resolved_tool_name = (
+                    _resolve_display_name_to_original(tool_name, [mcp_server]) if mcp_server else tool_name
+                )
+                sanitized_tool_name = strip_known_server_prefix(resolved_tool_name, mcp_server)
 
                 start_time = datetime.now()
                 logging_input = [
@@ -697,6 +704,10 @@ class LiteLLM_Proxy_MCP_Handler:
                 if request_tags:
                     logging_request_data["metadata"]["tags"] = request_tags
                 if user_api_key_auth is not None:
+                    from litellm.proxy.litellm_pre_call_utils import (
+                        LiteLLMProxyRequestSetup,
+                    )
+
                     LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
                         data=logging_request_data,
                         user_api_key_dict=user_api_key_auth,
@@ -741,7 +752,6 @@ class LiteLLM_Proxy_MCP_Handler:
                     "arguments": parsed_arguments,
                     "namespaced_tool_name": tool_name,
                 }
-                mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(tool_name)
                 if mcp_server:
                     mcp_info = mcp_server.mcp_info or {}
                     standard_logging_mcp_tool_call["mcp_server_name"] = (

litellm/responses/streaming_iterator.py

@@ -287,11 +287,12 @@ class BaseResponsesAPIStreamingIterator:
         end_time = datetime.now()
         if is_async:
             asyncio.create_task(
-                self.logging_obj.async_success_handler(
-                    result=logging_response,
+                self.logging_obj.dispatch_success_handlers(
+                    logging_response,
                     start_time=self.start_time,
                     end_time=end_time,
                     cache_hit=self._completed_response_cache_hit,
+                    prefer_async_handlers=True,
                 )
             )
         else:
@@ -302,14 +303,13 @@ class BaseResponsesAPIStreamingIterator:
                 end_time=end_time,
                 cache_hit=self._completed_response_cache_hit,
             )
-
-        executor.submit(
-            self.logging_obj.success_handler,
-            result=logging_response,
-            cache_hit=self._completed_response_cache_hit,
-            start_time=self.start_time,
-            end_time=end_time,
-        )
+            executor.submit(
+                self.logging_obj.success_handler,
+                result=logging_response,
+                cache_hit=self._completed_response_cache_hit,
+                start_time=self.start_time,
+                end_time=end_time,
+            )
         self._run_post_success_hooks(end_time=end_time)
 
     def _handle_logging_completed_response(self):
@@ -1136,7 +1136,6 @@ def _build_synthetic_response_events(
 # ---------------------------------------------------------------------------
 
 from litellm._logging import verbose_logger
-from litellm.litellm_core_utils.thread_pool_executor import executor as _ws_executor
 
 RESPONSES_WS_LOGGED_EVENT_TYPES = [
     "response.created",
@@ -1251,8 +1250,7 @@ class ResponsesWebSocketStreaming:
         if self.input_messages:
             self.logging_obj.model_call_details["messages"] = self.input_messages
         if self.messages:
-            asyncio.create_task(self.logging_obj.async_success_handler(self.messages))
-            _ws_executor.submit(self.logging_obj.success_handler, self.messages)
+            asyncio.create_task(self.logging_obj.dispatch_success_handlers(self.messages, prefer_async_handlers=True))
 
     async def backend_to_client(self) -> None:
         """Forward events from backend WebSocket to the client."""

litellm/responses/utils.py

@@ -204,6 +204,9 @@ class ResponsesAPIRequestUtils:
         if response_id is None:
             return responses_api_response
 
+        if ResponsesAPIRequestUtils._is_litellm_encoded_response_id(response_id):
+            return responses_api_response
+
         updated_id = ResponsesAPIRequestUtils._build_responses_api_response_id(
             model_id=model_id,
             custom_llm_provider=custom_llm_provider,
@@ -470,6 +473,14 @@ class ResponsesAPIRequestUtils:
                 response_id=response_id,
             )
 
+    @staticmethod
+    def _is_litellm_encoded_response_id(response_id: str) -> bool:
+        decoded_response_id = ResponsesAPIRequestUtils._decode_responses_api_response_id(response_id)
+        return (
+            decoded_response_id.get("model_id") is not None
+            or decoded_response_id.get("custom_llm_provider") is not None
+        )
+
     @staticmethod
     def get_model_id_from_response_id(response_id: Optional[str]) -> Optional[str]:
         """Get the model_id from the response_id"""

litellm/router.py

@@ -7394,8 +7394,7 @@ class Router:
             # deployment sharing the same backend model name.
             # Each deployment's full pricing is already stored under its
             # unique model_id above.
-            _custom_pricing_fields = CustomPricingLiteLLMParams.model_fields.keys()
-            _shared_model_info = {k: v for k, v in _model_info.items() if k not in _custom_pricing_fields}
+            _shared_model_info = CustomPricingLiteLLMParams.strip_custom_pricing_fields(_model_info)
             _existing_shared_mode = (cast(Optional[dict], litellm.model_cost.get(_model_name, {})) or {}).get("mode")
             _deployment_mode = _shared_model_info.get("mode")
             # Keep the built-in bridge mode stable for shared backend keys.
@@ -8059,8 +8058,7 @@ class Router:
         # deployment sharing the same backend model name.
         # Each deployment's full pricing is already stored under its
         # unique model_id above (when present).
-        _custom_pricing_fields = CustomPricingLiteLLMParams.model_fields.keys()
-        _shared_model_info = {k: v for k, v in _model_info_dict.items() if k not in _custom_pricing_fields}
+        _shared_model_info = CustomPricingLiteLLMParams.strip_custom_pricing_fields(_model_info_dict)
         _backend_alias_cost = {_model_name: _shared_model_info}
         if "responses/" in _model_name:
             _stripped_model_name = _model_name.replace("responses/", "")

litellm/secret_managers/base_secret_manager.py

@@ -1,3 +1,4 @@
+import re
 from abc import ABC, abstractmethod
 from typing import Any, Dict, Optional, Union
 
@@ -5,6 +6,20 @@ import httpx
 
 from litellm import verbose_logger
 
+_UNSAFE_SECRET_NAME_PATTERN = re.compile(r"(^|/)\.\.(/|$)|[\x00-\x1f\x7f-\x9f…

]")
+
+
+def raise_if_unsafe_secret_name(secret_name: str) -> None:
+    """
+    Validate a secret name before it is used by a secret manager integration.
+
+    Rejects ".." only as a path segment (bounded by "/" or the start/end of the
+    string, e.g. "../x", "x/..", or exactly ".."), not as a plain substring, so
+    names like "release-1.0..2" are not rejected.
+    """
+    if _UNSAFE_SECRET_NAME_PATTERN.search(secret_name):
+        raise ValueError(f"Invalid secret_name {secret_name!r}")
+
 
 class BaseSecretManager(ABC):
     """

litellm/secret_managers/cyberark_secret_manager.py

@@ -4,6 +4,7 @@ from typing import Any, Dict, Optional, Union
 from urllib.parse import quote
 
 import httpx
+import yaml
 
 import litellm
 from litellm._logging import verbose_logger
@@ -15,7 +16,7 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.proxy._types import KeyManagementSystem
 
-from .base_secret_manager import BaseSecretManager
+from .base_secret_manager import BaseSecretManager, raise_if_unsafe_secret_name
 from .main import str_to_bool
 
 
@@ -125,8 +126,11 @@ class CyberArkSecretManager(BaseSecretManager):
         """
         # In production, we'd check if the variable exists first
         # For now, we'll attempt to create it and ignore if it already exists
+        raise_if_unsafe_secret_name(secret_name)
         policy_url = f"{self.conjur_addr}/policies/{self.conjur_account}/policy/root"
-        policy_yaml = f"- !variable {secret_name}\n"
+        # Use a real YAML serializer to build the scalar safely.
+        quoted_name = yaml.safe_dump(secret_name, default_style='"').strip()
+        policy_yaml = f"- !variable {quoted_name}\n"
 
         try:
             client = _get_httpx_client(params={"ssl_verify": self.ssl_verify})

litellm/secret_managers/hashicorp_secret_manager.py

@@ -14,7 +14,7 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.proxy._types import KeyManagementSystem
 
-from .base_secret_manager import BaseSecretManager
+from .base_secret_manager import BaseSecretManager, raise_if_unsafe_secret_name
 
 
 class HashicorpSecretManager(BaseSecretManager):
@@ -220,6 +220,7 @@ class HashicorpSecretManager(BaseSecretManager):
         - With custom mount: http://127.0.0.1:8200/v1/kv/data/mykey
         - With path prefix: http://127.0.0.1:8200/v1/secret/data/myapp/mykey
         """
+        raise_if_unsafe_secret_name(secret_name)
         resolved_namespace = self._sanitize_path_component(namespace if namespace is not None else self.vault_namespace)
         resolved_mount = self._sanitize_path_component(mount_name if mount_name is not None else self.vault_mount_name)
         if resolved_mount is None:

litellm/types/management_endpoints/cache_settings_endpoints.py

@@ -40,6 +40,15 @@ CACHE_SETTINGS_FIELDS: List[CacheSettingsField] = [
         redis_type=None,
     ),
     # Common fields for all Redis types
+    CacheSettingsField(
+        field_name="url",
+        field_type="String",
+        field_value=None,
+        field_description="Full Redis/Valkey connection URL (e.g. redis://:password@host:6379/1). When set, it takes precedence over Host, Port, Username, Password, and Database Index.",
+        field_default=None,
+        ui_field_name="Redis URL",
+        redis_type=None,
+    ),
     CacheSettingsField(
         field_name="host",
         field_type="String",
@@ -58,6 +67,15 @@ CACHE_SETTINGS_FIELDS: List[CacheSettingsField] = [
         ui_field_name="Port",
         redis_type=None,
     ),
+    CacheSettingsField(
+        field_name="db",
+        field_type="Integer",
+        field_value=None,
+        field_description="Logical database index to isolate the cache (e.g. 1 for redis://host:6379/1)",
+        field_default=None,
+        ui_field_name="Database Index",
+        redis_type=None,
+    ),
     CacheSettingsField(
         field_name="password",
         field_type="String",

litellm/types/mcp.py

@@ -38,8 +38,16 @@ class MCPAuth(str, enum.Enum):
     aws_sigv4 = "aws_sigv4"
     token = "token"
     oauth2_token_exchange = "oauth2_token_exchange"
+    true_passthrough = "true_passthrough"
+    oauth_delegate = "oauth_delegate"
 
 
+# RFC 8693 default subject_token_type. A NULL column / omitted config key means
+# "use this default"; it is applied at every egress build site via this single
+# constant rather than a DB-level DEFAULT (Prisma writes explicit values on
+# insert, so a column default would rarely apply anyway).
+DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token"
+
 # MCP Literals
 MCPTransportType = Literal[MCPTransport.sse, MCPTransport.http, MCPTransport.stdio]
 MCPSpecVersionType = Literal[MCPSpecVersion.nov_2024, MCPSpecVersion.mar_2025, MCPSpecVersion.jun_2025]
@@ -54,6 +62,8 @@ MCPAuthType = Optional[
         MCPAuth.aws_sigv4,
         MCPAuth.token,
         MCPAuth.oauth2_token_exchange,
+        MCPAuth.true_passthrough,
+        MCPAuth.oauth_delegate,
     ]
 ]
 
@@ -122,18 +132,31 @@ class MCPCredentials(TypedDict, total=False):
 
     audience: Optional[str]
     """
-    Target audience for OAuth 2.0 Token Exchange (RFC 8693)
+    Target audience for OAuth 2.0 Token Exchange (RFC 8693).
+
+    Legacy input shape: this setting has a dedicated ``audience`` column, which is
+    authoritative. A value sent here is accepted for back-compat (the pre-column
+    REST shape, released since 2026-05), lifted into the column on write, and
+    stripped from the stored blob. Prefer the top-level request field.
     """
 
     token_exchange_endpoint: Optional[str]
     """
-    IDP token endpoint for OAuth 2.0 Token Exchange (RFC 8693)
+    IDP token endpoint for OAuth 2.0 Token Exchange (RFC 8693).
+
+    Legacy input shape: lifted into the dedicated ``token_exchange_endpoint``
+    column on write and stripped from the stored blob; the column is
+    authoritative. Prefer the top-level request field.
     """
 
     subject_token_type: Optional[str]
     """
     Subject token type for OAuth 2.0 Token Exchange (RFC 8693).
-    Default: urn:ietf:params:oauth:token-type:access_token
+    Default: DEFAULT_SUBJECT_TOKEN_TYPE (urn:ietf:params:oauth:token-type:access_token).
+
+    Legacy input shape: lifted into the dedicated ``subject_token_type`` column on
+    write and stripped from the stored blob; the column is authoritative. Prefer
+    the top-level request field.
     """
 
     token_endpoint_auth_method: Optional[MCPTokenEndpointAuthMethod]
@@ -147,6 +170,10 @@ class MCPCredentials(TypedDict, total=False):
     Token exchange wire dialect: "rfc8693" (default, the standard token-exchange grant) or
     "entra_obo" (Microsoft Entra On-Behalf-Of, the RFC 7523 jwt-bearer grant + requested_token_use
     extension). Not a secret; stored unencrypted.
+
+    Legacy input shape: lifted into the dedicated ``token_exchange_profile`` column on
+    write and stripped from the stored blob; the column is authoritative. Prefer the
+    top-level request field.
     """
 
 

litellm/types/mcp_server/mcp_server_manager.py

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Literal, Optional
 from pydantic import BaseModel, ConfigDict
 
 from litellm.types.mcp import (
+    DEFAULT_SUBJECT_TOKEN_TYPE,
     MCPAuth,
     MCPAuthType,
     MCPTokenEndpointAuthMethod,
@@ -68,7 +69,7 @@ class MCPServer(BaseModel):
     # Token Exchange (OBO) fields
     token_exchange_endpoint: Optional[str] = None
     audience: Optional[str] = None
-    subject_token_type: str = "urn:ietf:params:oauth:token-type:access_token"
+    subject_token_type: str = DEFAULT_SUBJECT_TOKEN_TYPE
     # Wire dialect: "rfc8693" (standard token-exchange grant) or "entra_obo" (Microsoft Entra
     # On-Behalf-Of, the RFC 7523 jwt-bearer grant + requested_token_use extension)
     token_exchange_profile: str = "rfc8693"
@@ -151,6 +152,18 @@ class MCPServer(BaseModel):
         """True if this is an OAuth2 server that relies on per-user tokens (no client_credentials)."""
         return self.auth_type == MCPAuth.oauth2 and not self.has_client_credentials
 
+    @property
+    def is_true_passthrough(self) -> bool:
+        """True for the transparent-proxy mode: LiteLLM performs no admission auth and forwards the
+        client's ``Authorization`` to the upstream unchanged."""
+        return self.auth_type == MCPAuth.true_passthrough
+
+    @property
+    def is_oauth_delegate(self) -> bool:
+        """True for the delegated-upstream-OAuth mode: LiteLLM still admits the caller (API key / SSO /
+        JWT) but forwards the caller's separate upstream ``Authorization`` unchanged, minting nothing."""
+        return self.auth_type == MCPAuth.oauth_delegate
+
     @property
     def requires_per_user_auth(self) -> bool:
         """
@@ -166,6 +179,9 @@ class MCPServer(BaseModel):
         if self.needs_user_oauth_token:
             return True
 
+        if self.is_true_passthrough or self.is_oauth_delegate:
+            return True
+
         # PAT passthrough: auth_type is none but extra_headers includes auth headers
         if self.auth_type == MCPAuth.none and self.extra_headers:
             auth_header_names = {"authorization", "x-api-key", "api-key", "apikey"}

litellm/types/utils.py

@@ -3048,6 +3048,17 @@ class CustomPricingLiteLLMParams(BaseModel):
     regional_processing_uplift_multiplier_eu: Optional[float] = None
     regional_processing_uplift_multiplier_us: Optional[float] = None
 
+    @classmethod
+    def strip_custom_pricing_fields(cls, model_info: Dict[str, Any]) -> Dict[str, Any]:
+        """Return a copy of ``model_info`` without per-deployment custom pricing fields.
+
+        Used when registering a deployment's info under the shared
+        ``{provider}/{model}`` key in ``litellm.model_cost``, so one deployment's
+        pricing overrides don't pollute sibling deployments that share the same
+        backend model. Full pricing stays under the deployment's unique model id.
+        """
+        return {k: v for k, v in model_info.items() if k not in cls.model_fields}
+
 
 # Server-controlled fields that bound or drive an interceptor's agentic loop
 # (depth, cycle fingerprints, ceiling, code-interpreter sandbox state). Listed

litellm/utils.py

@@ -2619,8 +2619,9 @@ _CACHE_PRICING_FIELDS = (
 
 def _resolve_builtin_model_cost_entry(key: str, provider: str) -> Optional[Dict[str, Any]]:
     """Best-effort lookup of a built-in ``model_cost`` entry for a custom key
-    whose shape ``get_model_info`` cannot resolve (double provider prefixes
-    like ``bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region aliases).
+    whose shape ``get_model_info`` cannot resolve (repeated provider prefixes
+    like ``bedrock/bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region
+    aliases).
 
     Returns a copy of the matching entry so the caller can inherit its defaults
     (most importantly cache pricing) without mutating the shared built-in.
@@ -5052,9 +5053,9 @@ def _get_model_info_from_generalization(
     candidates = [
         potential_model_names["combined_model_name"],
         model,
+        potential_model_names["split_model"],
         potential_model_names["combined_stripped_model_name"],
         potential_model_names["stripped_model_name"],
-        potential_model_names["split_model"],
     ]
     for candidate in candidates:
         generalized_info = match_fallback_generalization(candidate)
@@ -5094,6 +5095,11 @@ def _get_potential_model_names(
             stripped_model_name,
         )
 
+    if custom_llm_provider in ("bedrock", "bedrock_converse"):
+        from litellm.llms.bedrock.common_utils import strip_bedrock_routing_prefix
+
+        split_model = strip_bedrock_routing_prefix(split_model)
+
     return PotentialModelNamesAndCustomLLMProvider(
         split_model=split_model,
         combined_model_name=combined_model_name,
@@ -5261,9 +5267,9 @@ def _get_model_info_helper(
             Check if: (in order of specificity)
             1. 'custom_llm_provider/model' in litellm.model_cost. Checks "groq/llama3-8b-8192" if model="llama3-8b-8192" and custom_llm_provider="groq"
             2. 'model' in litellm.model_cost. Checks "gemini-1.5-pro-002" in  litellm.model_cost if model="gemini-1.5-pro-002" and custom_llm_provider=None
-            3. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
-            4. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
-            5. 'split_model' in litellm.model_cost. Checks "llama3-8b-8192" in litellm.model_cost if model="groq/llama3-8b-8192"
+            3. 'split_model' in litellm.model_cost. Checks "au.anthropic.claude-opus-4-8" in litellm.model_cost if model="bedrock/au.anthropic.claude-opus-4-8"
+            4. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
+            5. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
             """
 
             _model_info: Optional[Dict[str, Any]] = None
@@ -5289,6 +5295,16 @@ def _get_model_info_helper(
                         custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
+            if _model_info is None:
+                _matched_key = _get_model_cost_key(split_model)
+                if _matched_key is not None:
+                    key = _matched_key
+                    _model_info = _get_model_info_from_model_cost(key=cast(str, key))
+                    if not _check_provider_match(
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
+                    ):
+                        _model_info = None
             if _model_info is None:
                 _matched_key = _get_model_cost_key(combined_stripped_model_name)
                 if _matched_key is not None:
@@ -5309,16 +5325,6 @@ def _get_model_info_helper(
                         custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
-            if _model_info is None:
-                _matched_key = _get_model_cost_key(split_model)
-                if _matched_key is not None:
-                    key = _matched_key
-                    _model_info = _get_model_info_from_model_cost(key=cast(str, key))
-                    if not _check_provider_match(
-                        model_info=_model_info,
-                        custom_llm_provider=model_cost_custom_llm_provider,
-                    ):
-                        _model_info = None
 
             if _model_info is None:
                 generalization = _get_model_info_from_generalization(

model_prices_and_context_window.json

@@ -23661,6 +23661,76 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "gpt-realtime-2.1": {
+        "cache_creation_input_audio_token_cost": 4e-07,
+        "cache_read_input_audio_token_cost": 4e-07,
+        "cache_read_input_token_cost": 4e-07,
+        "input_cost_per_audio_token": 3.2e-05,
+        "input_cost_per_image": 5e-06,
+        "input_cost_per_token": 4e-06,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "max_tokens": 32000,
+        "mode": "chat",
+        "output_cost_per_audio_token": 6.4e-05,
+        "output_cost_per_token": 2.4e-05,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/realtime"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
+    "gpt-realtime-2.1-mini": {
+        "cache_creation_input_audio_token_cost": 3e-07,
+        "cache_read_input_audio_token_cost": 3e-07,
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_image": 8e-07,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "openai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "supported_endpoints": [
+            "/v1/realtime"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
     "gpt-realtime-mini": {
         "cache_creation_input_audio_token_cost": 3e-07,
         "cache_read_input_audio_token_cost": 3e-07,
@@ -38233,6 +38303,48 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "xai/grok-4.5": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 1e-06,
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_200k_tokens": 4e-06,
+        "litellm_provider": "xai",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 1.2e-05,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "xai/grok-4.5-latest": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 1e-06,
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_200k_tokens": 4e-06,
+        "litellm_provider": "xai",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 1.2e-05,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "xai/grok-beta": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "xai",

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.92.0"
+version = "1.93.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.14"
@@ -63,7 +63,7 @@ proxy = [
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.26.0,<2.0",
     "litellm-proxy-extras==0.4.75",
-    "litellm-enterprise==0.1.47",
+    "litellm-enterprise==0.1.48",
     "RestrictedPython>=8.1,<9.0",
     "rich>=13.9.4,<14.0",
     "polars>=1.38.1,<2.0",
@@ -279,7 +279,7 @@ members = ["enterprise", "litellm-proxy-extras"]
 profile = "black"
 
 [tool.commitizen]
-version = "1.92.0"
+version = "1.93.0"
 version_files = [
     "pyproject.toml:^version",
 ]

schema.prisma

@@ -329,6 +329,12 @@ model LiteLLM_MCPServerTable {
   token_url          String?
   registration_url   String?
   oauth2_flow        String?
+  token_exchange_endpoint String?
+  // Named for the RFC 8693 "audience" token-exchange request parameter (that flow only).
+  // RFC 8707 resource indicators are a separate concept, named "resource" in the v2 egress types.
+  audience           String?
+  subject_token_type String?
+  token_exchange_profile String?
   allow_all_keys Boolean @default(false)
   available_on_public_internet Boolean @default(true)
   delegate_auth_to_upstream Boolean @default(false)

terraform/provider/.gitignore

@@ -0,0 +1,71 @@
+# Local .terraform directories
+**/.terraform/*
+test_litellm/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data
+*.tfvars
+!*.tfvars.example
+
+# Ignore override files as they are usually used to override resources locally
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
+# Binary files
+terraform-provider-litellm
+
+# IDE and editor files
+.idea/
+*.swp
+*.swo
+.vscode/
+*.sublime-workspace
+*.sublime-project
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Go specific
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+*.test
+*.out
+go.work
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+
+# Log files
+*.log
+
+# Environment files
+.env

terraform/provider/.goreleaser.yml

@@ -0,0 +1,81 @@
+# Visit https://goreleaser.com for documentation on how to customize this
+# behavior.
+version: 2
+before:
+  hooks:
+    # this is just an example and not a requirement for provider building/publishing
+    - go mod tidy
+builds:
+- env:
+    # goreleaser does not work with CGO, it could also complicate
+    # usage by users in CI/CD systems like HCP Terraform where
+    # they are unable to install libraries.
+    - CGO_ENABLED=0
+  mod_timestamp: '{{ .CommitTimestamp }}'
+  flags:
+    - -trimpath
+  ldflags:
+    - '-s -w -X main.version={{.Version}} -X main.commit={{.Commit}}'
+  goos:
+    - freebsd
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+    - '386'
+    - arm
+    - arm64
+  ignore:
+    # macOS doesn't support 32-bit anymore
+    - goos: darwin
+      goarch: '386'
+    # Windows ARM is uncommon for Terraform usage
+    - goos: windows
+      goarch: arm
+    - goos: windows
+      goarch: arm64
+    # FreeBSD ARM is rarely used
+    - goos: freebsd
+      goarch: arm
+    - goos: freebsd
+      goarch: arm64
+  # This builds the following key targets for Terraform users:
+  # - linux/amd64 (most common CI/CD)
+  # - linux/arm64 (Graviton, ARM-based CI)
+  # - linux/386 (legacy 32-bit systems)
+  # - linux/arm (Raspberry Pi, etc.)
+  # - darwin/amd64 (Intel Macs)
+  # - darwin/arm64 (Apple Silicon Macs)
+  # - windows/amd64 (Windows desktops)
+  # - freebsd/amd64, freebsd/386 (FreeBSD servers)
+  binary: '{{ .ProjectName }}_v{{ .Version }}'
+archives:
+- format: zip
+  name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}'
+checksum:
+  extra_files:
+    - glob: 'terraform-registry-manifest.json'
+      name_template: '{{ .ProjectName }}_{{ .Version }}_manifest.json'
+  name_template: '{{ .ProjectName }}_{{ .Version }}_SHA256SUMS'
+  algorithm: sha256
+signs:
+  - artifacts: checksum
+    args:
+      # if you are using this in a GitHub action or some other automated pipeline, you 
+      # need to pass the batch flag to indicate its not interactive.
+      - "--batch"
+      - "--local-user"
+      - "{{ .Env.GPG_FINGERPRINT }}" # set this environment variable for your signing key
+      - "--output"
+      - "${signature}"
+      - "--detach-sign"
+      - "${artifact}"
+release:
+  extra_files:
+    - glob: 'terraform-registry-manifest.json'
+      name_template: '{{ .ProjectName }}_{{ .Version }}_manifest.json'
+  # If you want to manually examine the release before its live, uncomment this line:
+  # draft: true
+changelog:
+  disable: true

terraform/provider/CHANGELOG.md

@@ -0,0 +1,294 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Fixed
+
+- **organization**: Send `PATCH` instead of `POST` to `/organization/update` and `/organization/member_update`, matching the methods the LiteLLM proxy serves; organization and organization member updates previously failed with a 405
+
+### Changed
+
+- The provider source of truth moved to `terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm); this repository is now a release mirror. CI in the monorepo statically audits every endpoint the provider calls against the proxy's OpenAPI schema on every change
+
+## [0.2.2] - 2026-05-13
+
+### Fixed
+
+- **key**: Include `tags` in `UpdateKey` payload so tag changes on an existing `litellm_key` are applied on update instead of being silently dropped (#41)
+
+## [0.2.1] - 2026-04-13
+
+### Fixed
+
+- **team, organization**: Use pointer types for `tpm_limit`, `rpm_limit`, and `max_budget` to prevent zero-value diffs on every `terraform plan` when these fields are not configured (#31)
+
+## [0.2.0] - 2026-04-03
+
+### ⚠️ Breaking Changes
+
+#### `litellm_key`: API keys are no longer stored in Terraform state
+
+**Why this change?** Storing raw API keys in Terraform state is a security risk — state files are often stored in S3, Terraform Cloud, or other backends where the key could be exposed even with encryption at rest. This release eliminates that risk entirely.
+
+**What changed:**
+- The `key` attribute is now **write-only** — available during `terraform apply` so you can pipe it to a secrets manager, but never persisted to state
+- The resource ID has changed from the raw key value to its **SHA-256 hash (`token_id`)** — safe to store in state, cannot be used to authenticate
+- **Requires Terraform 1.11+**
+
+**Migration steps for existing `litellm_key` resources:**
+
+1. Find the `token_id` for each key via the LiteLLM UI or `GET /key/info?key=<your-key>`
+2. Remove the old resource from state:
+   ```
+   terraform state rm litellm_key.example
+   ```
+3. Re-import using the token_id:
+   ```
+   terraform import litellm_key.example <token_id>
+   ```
+
+> ⚠️ After upgrading, you cannot retrieve the raw key from state. Make sure you have the key value stored somewhere safe before migrating, or plan to rotate the key after re-import.
+
+**Security best practice:** Since the key is only available during the initial `terraform apply`, pipe it directly to a secrets manager:
+
+```hcl
+resource "aws_ssm_parameter" "litellm_key" {
+  name  = "/myapp/litellm-key"
+  type  = "SecureString"
+  value = litellm_key.example.key
+}
+```
+
+### Fixed
+
+- **key**: API key is no longer stored in Terraform state. The `key` attribute is now write-only and `token_id` is used as the resource ID (#27)
+- **model**: Handle eventual consistency in model reads post-create (#26)
+
+## [0.1.2] - 2026-02-17
+
+### Added
+- **Documentation**: Added RELEASING.md with comprehensive release process documentation
+  - GPG key setup instructions
+  - Step-by-step release workflow
+  - Troubleshooting guide
+  - Security best practices
+
+## [0.1.1] - 2026-02-11
+
+### Added
+- **New Model Modes**: Added support for `audio_speech` and `rerank` model modes
+  - `audio_speech`: For text-to-speech models (e.g., Gemini TTS, OpenAI TTS)
+  - `rerank`: For reranking/semantic ranking models (e.g., Cohere Rerank, Vertex AI Semantic Ranker)
+
+### Fixed
+- Implemented exponential backoff for credential reads
+- Only include cost fields when explicitly set in model resource
+- Added litellm_credential_name support
+
+## [0.3.14] - 2025-08-24
+
+### Added
+- **Enhanced JSON Parsing**: Added support for JSON string parsing in `additional_litellm_params`
+  - JSON objects and arrays (starting with `{` or `[`) are now automatically parsed
+  - Maintains backward compatibility with existing string-to-type conversion
+  - Enables complex nested parameter configurations
+- **Parameter Dropping Feature**: Added `additional_drop_params` special parameter
+  - Allows removal of unwanted parameters from final `litellm_params` before API submission
+  - Specified as JSON array string: `"additional_drop_params" = "[\"reasoningEffort\"]"`
+  - Useful for overriding or removing built-in parameters when needed
+- **Enhanced Examples**: Updated `examples/model_additional_params.tf` with comprehensive JSON parsing examples
+  - Demonstrates all supported value types (boolean, integer, float, string, JSON objects/arrays)
+  - Includes real-world Azure model configuration with parameter dropping
+  - Shows both simple and complex use cases
+
+### Changed
+- **Documentation Enhancement**: Updated `docs/resources/model.md` with detailed JSON parsing documentation
+  - Added comprehensive explanation of conversion rules and behavior
+  - Included special `additional_drop_params` parameter documentation
+  - Enhanced examples showing all supported parameter types and JSON parsing capabilities
+
+### Technical Details
+- Enhanced parameter processing logic in `createOrUpdateModel()` function
+- Added JSON detection and parsing for string values starting with `[` or `{`
+- Implemented parameter filtering system for `additional_drop_params`
+- Maintains full backward compatibility with existing configurations
+
+## [0.3.13] - 2025-08-24
+
+### Changed
+- Documentation: Performed a documentation audit and improvements across resources and data-sources. Added missing argument references, clarified types/defaults, documented implementation behaviors (e.g., additional_litellm_params parsing and state-preservation), and added an `examples/` directory with runnable HCL examples (starting with `examples/model_additional_params.tf`).
+- Docs: Updated `docs/resources/model.md` with missing fields (`vertex_*`, pixel/second cost fields, and `additional_litellm_params`) and added conversion rules and an example.
+- Docs Index: Added references to the new `examples/` directory in `docs/index.md`.
+
+## [0.3.12] - 2025-08-13
+
+### Added
+- **New AWS Parameters**: Added `aws_session_name` and `aws_role_name` to model resource for cross-account access scenarios
+  - Support for AWS session names in cross-account access configurations
+  - Support for AWS IAM role names for cross-account access
+  - Enhanced AWS Bedrock integration capabilities
+
+### Changed
+- **Documentation Overhaul**: Comprehensive update to all provider documentation
+  - Updated provider source references from `bitop/litellm` to `registry.terraform.io/ncecere/litellm`
+  - Consolidated all scattered example files into organized documentation structure
+  - Enhanced all resource documentation with multiple real-world examples
+  - Added comprehensive cross-resource integration examples
+- **Vector Store Documentation**: Updated to reflect only officially supported LiteLLM providers
+  - Removed unsupported providers (Pinecone, Weaviate, Chroma, Qdrant, Milvus, FAISS)
+  - Added accurate examples for supported providers: AWS Bedrock Knowledge Bases, OpenAI Vector Stores, Azure Vector Stores, Vertex AI RAG Engine, PG Vector
+  - Updated provider-specific parameters with correct configurations
+  - Added references to official LiteLLM documentation
+- **Project Organization**: Cleaned up project structure
+  - Removed scattered example files from root directory
+  - Consolidated all examples into comprehensive documentation
+  - Updated README.md to reflect current capabilities and structure
+
+### Fixed
+- Corrected vector store provider documentation to match LiteLLM's official capabilities
+- Updated all documentation links and references for accuracy
+
+## [0.3.11] - 2025-08-10
+
+### Added
+- **New Resource**: `litellm_credential` - Manage credentials for secure authentication
+  - Support for storing sensitive credential values (API keys, tokens, etc.)
+  - Non-sensitive credential information storage
+  - Model ID association for credentials
+  - Secure handling of sensitive data with Terraform's sensitive attribute
+- **New Resource**: `litellm_vector_store` - Manage vector stores for embeddings and RAG
+  - Support for multiple vector store providers (Pinecone, Weaviate, Chroma, Qdrant, etc.)
+  - Integration with credential management for secure authentication
+  - Configurable metadata and provider-specific parameters
+  - Full CRUD operations for vector store lifecycle management
+- **New Data Source**: `litellm_credential` - Retrieve information about existing credentials
+  - Read-only access to credential metadata (sensitive values excluded for security)
+  - Support for model ID filtering
+  - Cross-stack and cross-configuration referencing capabilities
+- **New Data Source**: `litellm_vector_store` - Retrieve information about existing vector stores
+  - Complete vector store information retrieval
+  - Support for monitoring, validation, and cross-referencing use cases
+  - Metadata-based conditional logic support
+- Enhanced API response handling for credential and vector store operations
+- Comprehensive documentation and examples for new resources and data sources
+- Example Terraform configurations for common use cases
+
+### Changed
+- Extended `utils.go` with specialized API response handlers for credentials and vector stores
+- Updated provider configuration to include new resources and data sources
+- Enhanced error handling for credential and vector store not found scenarios
+
+## [0.3.10] - 2025-08-10
+
+### Added
+- **New Resource**: `litellm_mcp_server` - Manage MCP (Model Context Protocol) servers
+  - Support for HTTP, SSE, and stdio transport types
+  - Configurable authentication types (none, bearer, basic)
+  - MCP access groups for permission management
+  - Cost tracking configuration for MCP tools
+  - Environment variables and command arguments for stdio transport
+  - Health check status monitoring
+  - Comprehensive documentation and examples
+
+### Changed
+- Updated provider to support MCP server management functionality
+- Enhanced API response handling for MCP-specific operations
+
+## [0.3.9] - 2025-08-10
+
+### Fixed
+- Fixed issue where omitting `budget_duration` in key resource caused API error "Invalid duration format"
+- Added missing `omitempty` JSON tag to `BudgetDuration` field in Key struct to prevent sending empty strings to API
+
+## [0.3.8] - 2025-08-08
+
+### Added
+- Added `additional_litellm_params` field to model resource for custom parameters beyond standard ones
+- Support for passing custom parameters like `drop_params`, `timeout`, `max_retries`, `organization`, etc.
+- Automatic type conversion for string values to appropriate types (boolean, integer, float)
+- Full backward compatibility with existing model configurations
+- Comprehensive example demonstrating various use cases with different providers
+
+## [0.3.7] - 2025-08-08
+
+### Fixed
+- Fixed issue where changing max_budget_in_team didn't update existing team members with new budget
+- Added budget change detection using d.HasChange to update ALL existing members when budget changes
+- Implemented tracking to avoid duplicate API calls for members already updated
+- Enhanced debug logging for budget update operations
+
+## [0.3.6] - 2025-08-08
+
+### Fixed
+- Fixed issue where models deleted from LiteLLM proxy caused terraform plan to fail instead of planning recreation
+- Enhanced ErrorResponse struct to properly parse LiteLLM proxy error format with Detail field
+- Improved isModelNotFoundError function to detect "not found on litellm proxy" messages in Detail.Error field
+
+## [0.3.5] - 2025-08-08
+
+### Fixed
+- Fixed team member update behavior to use member_update endpoint instead of delete/re-add
+- Restored team_member_permissions functionality to litellm_team resource
+- Enhanced team resource with proper permissions management endpoints
+
+## [0.3.0] - 2025-04-23
+
+### Fixed
+- Implemented retry mechanism with exponential backoff for model read operations
+- Added detailed logging for retry attempts
+- Improved error handling for "model not found" errors
+
+## [0.2.9] - 2025-04-23
+
+### Fixed
+- Increased delay after model creation from 2 to 5 seconds to fix "model not found" errors
+- Added logging to confirm delay is working properly
+
+## [0.2.8] - 2025-04-23
+
+### Fixed
+- Added delay after model creation to fix "model not found" errors when the LiteLLM proxy hasn't fully registered the model yet
+
+## [0.2.7] - 2025-04-23
+
+### Fixed
+- Fixed issue where `thinking_enabled` and `merge_reasoning_content_in_choices` values were not being preserved in state, causing Terraform to want to modify them on every run
+
+## [0.2.6] - 2025-03-13
+
+### Added
+- Added new `merge_reasoning_content_in_choices` option to model resource
+
+## [0.2.5] - 2025-03-13
+
+### Fixed
+- Fixed issue where `thinking_budget_tokens` was being added to models that don't have `thinking_enabled = true`
+
+## [0.2.4] - 2025-03-13
+
+### Added
+- Added new `thinking` capability to model resource with configurable parameters:
+  - `thinking_enabled` - Boolean to enable/disable thinking capability (default: false)
+  - `thinking_budget_tokens` - Integer to set token budget for thinking (default: 1024)
+
+## [0.2.2] - 2025-02-06
+
+### Added
+- Added new `reasoning_effort` parameter to model resource with values: "low", "medium", "high"
+- Added "chat" mode to model resource
+
+### Changed
+- Updated model mode options to: "completion", "embedding", "image_generation", "chat", "moderation", "audio_transcription"
+
+## [1.0.0] - 2024-01-17
+
+### Added
+- Initial release of the LiteLLM Terraform Provider
+- Support for managing LiteLLM models
+- Support for managing teams and team members
+- Comprehensive documentation for all resources

terraform/provider/LICENSE

@@ -0,0 +1,35 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source

terraform/provider/Makefile

@@ -0,0 +1,32 @@
+HOSTNAME=registry.terraform.io
+NAMESPACE=local
+NAME=litellm
+VERSION=1.0.0
+OS_ARCH=darwin_amd64
+
+default: install
+
+build:
+	go build -o terraform-provider-${NAME}
+
+install: build
+	mkdir -p ~/.terraform.d/plugins/${HOSTNAME}/${NAMESPACE}/${NAME}/${VERSION}/${OS_ARCH}
+	mv terraform-provider-${NAME} ~/.terraform.d/plugins/${HOSTNAME}/${NAMESPACE}/${NAME}/${VERSION}/${OS_ARCH}/terraform-provider-${NAME}_v${VERSION}
+
+test:
+	go test ./...
+
+fmt:
+	go fmt ./...
+
+vet:
+	go vet ./...
+
+lint:
+	golangci-lint run
+
+clean:
+	rm -f terraform-provider-${NAME}
+	rm -rf ~/.terraform.d/plugins/${HOSTNAME}/${NAMESPACE}/${NAME}/${VERSION}
+
+.PHONY: build install test fmt vet lint clean

terraform/provider/README.md

@@ -0,0 +1,223 @@
+# LiteLLM Terraform Provider
+
+This Terraform provider allows you to manage LiteLLM resources through Infrastructure as Code. It provides support for managing models, teams, team members, and API keys via the LiteLLM REST API.
+
+## Source of truth
+
+This directory (`terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm)) is the source of truth for the provider. [BerriAI/terraform-provider-litellm](https://github.com/BerriAI/terraform-provider-litellm) is a thin release mirror that the public Terraform Registry ingests from; do not open PRs there. Changes land here, where CI builds the provider, runs its tests, and statically audits every endpoint the provider calls against the proxy's generated OpenAPI schema (`tools/endpointaudit/`), so the provider cannot drift from the LiteLLM API silently. Releases are published by mirroring this directory into the split repo and tagging it, which triggers the goreleaser workflow there (see `RELEASING.md`)
+
+## Features
+
+- Manage LiteLLM model configurations
+- Associate models with specific teams
+- Create and manage teams
+- Configure team members and their permissions
+- Set usage limits and budgets
+- Control access to specific models
+- Specify model modes (e.g., completion, embedding, image generation)
+- Manage API keys with fine-grained controls
+- Support for reasoning effort configuration in the model resource
+
+## Requirements
+
+- [Terraform](https://www.terraform.io/downloads.html) >= 0.13.x
+- [Go](https://golang.org/doc/install) >= 1.16 (for development)
+
+## Using the Provider
+
+To use the LiteLLM provider in your Terraform configuration, you need to declare it in the <code>terraform</code> block:
+
+```hcl
+terraform {
+  required_providers {
+    litellm = {
+      source  = "BerriAI/litellm"
+      version = "~> 0.1.1" #HERE UPDATE VERSION ACCORDINGLY
+    }
+  }
+}
+
+provider "litellm" {
+  api_base = var.litellm_api_base
+  api_key  = var.litellm_api_key
+}
+```
+
+Then, you can use the provider to manage LiteLLM resources. Here's an example of creating a model configuration:
+
+```hcl
+resource "litellm_model" "gpt4" {
+  model_name          = "gpt-4-proxy"
+  custom_llm_provider = "openai"
+  model_api_key       = var.openai_api_key
+  model_api_base      = "https://api.openai.com/v1"
+  base_model          = "gpt-4"
+  tier                = "paid"
+  mode                = "chat"
+  reasoning_effort    = "medium"  # Optional: "low", "medium", or "high"
+  
+  input_cost_per_million_tokens  = 30.0
+  output_cost_per_million_tokens = 60.0
+}
+```
+
+For full details on the <code>litellm_model</code> resource, see the [model resource documentation](docs/resources/model.md).
+
+Here's an example of creating an API key with various options:
+
+```hcl
+resource "litellm_key" "example_key" {
+  models               = ["gpt-4", "claude-3.5-sonnet"]
+  max_budget           = 100.0
+  user_id              = "user123"
+  team_id              = "team456"
+  max_parallel_requests = 5
+  tpm_limit            = 1000
+  rpm_limit            = 60
+  budget_duration      = "monthly"
+  key_alias            = "prod-key-1"
+  duration             = "30d"
+  metadata             = {
+    environment = "production"
+  }
+  allowed_cache_controls = ["no-cache", "max-age=3600"]
+  soft_budget          = 80.0
+  aliases              = {
+    "gpt-4" = "gpt4"
+  }
+  config               = {
+    default_model = "gpt-4"
+  }
+  permissions          = {
+    can_create_keys = "true"
+  }
+  model_max_budget     = {
+    "gpt-4" = 50.0
+  }
+  model_rpm_limit      = {
+    "claude-3.5-sonnet" = 30
+  }
+  model_tpm_limit      = {
+    "gpt-4" = 500
+  }
+  guardrails           = ["content_filter", "token_limit"]
+  blocked              = false
+  tags                 = ["production", "api"]
+}
+```
+
+The <code>litellm_key</code> resource supports the following options:
+
+- <code>models</code>: List of allowed models for this key
+- <code>max_budget</code>: Maximum budget for the key
+- <code>user_id</code> and <code>team_id</code>: Associate the key with a user and team
+- <code>max_parallel_requests</code>: Limit concurrent requests
+- <code>tpm_limit</code> and <code>rpm_limit</code>: Set tokens and requests per minute limits
+- <code>budget_duration</code>: Specify budget duration (e.g., "monthly", "weekly")
+- <code>key_alias</code>: Set a friendly name for the key
+- <code>duration</code>: Set the key's validity period
+- <code>metadata</code>: Add custom metadata to the key
+- <code>allowed_cache_controls</code>: Specify allowed cache control directives
+- <code>soft_budget</code>: Set a soft budget limit
+- <code>aliases</code>: Define model aliases
+- <code>config</code>: Set configuration options
+- <code>permissions</code>: Specify key permissions
+- <code>model_max_budget</code>, <code>model_rpm_limit</code>, <code>model_tpm_limit</code>: Set per-model limits
+- <code>guardrails</code>: Apply specific guardrails to the key
+- <code>blocked</code>: Flag to block/unblock the key
+- <code>tags</code>: Add tags for organization and filtering
+
+For full details on the <code>litellm_key</code> resource, see the [key resource documentation](docs/resources/key.md).
+
+### Available Resources
+
+- <code>litellm_model</code>: Manage model configurations. [Documentation](docs/resources/model.md)
+- <code>litellm_team</code>: Manage teams. [Documentation](docs/resources/team.md)
+- <code>litellm_team_member</code>: Manage team members. [Documentation](docs/resources/team_member.md)
+- <code>litellm_team_member_add</code>: Add multiple members to teams. [Documentation](docs/resources/team_member_add.md)
+- <code>litellm_key</code>: Manage API keys. [Documentation](docs/resources/key.md)
+- <code>litellm_mcp_server</code>: Manage MCP (Model Context Protocol) servers. [Documentation](docs/resources/mcp_server.md)
+- <code>litellm_credential</code>: Manage credentials for secure authentication. [Documentation](docs/resources/credential.md)
+- <code>litellm_vector_store</code>: Manage vector stores for embeddings and RAG. [Documentation](docs/resources/vector_store.md)
+
+### Available Data Sources
+
+- <code>litellm_credential</code>: Retrieve information about existing credentials. [Documentation](docs/data-sources/credential.md)
+- <code>litellm_vector_store</code>: Retrieve information about existing vector stores. [Documentation](docs/data-sources/vector_store.md)
+
+## Development
+
+### Project Structure
+
+The project is organized as follows:
+
+```
+terraform-provider-litellm/
+├── litellm/
+│   ├── provider.go
+│   ├── resource_model.go
+│   ├── resource_model_crud.go
+│   ├── resource_team.go
+│   ├── resource_team_member.go
+│   ├── resource_key.go
+│   ├── resource_key_utils.go
+│   ├── types.go
+│   └── utils.go
+├── main.go
+├── go.mod
+├── go.sum
+├── Makefile
+└── ...
+```
+
+### Building the Provider
+
+1. Clone the repository:
+```sh
+git clone https://github.com/your-username/terraform-provider-litellm.git
+```
+
+2. Enter the repository directory:
+```sh
+cd terraform-provider-litellm
+```
+
+3. Build and install the provider:
+```sh
+make install
+```
+
+### Development Commands
+
+The Makefile provides several useful commands for development:
+
+- `make build`: Builds the provider
+- `make install`: Builds and installs the provider
+- `make test`: Runs the test suite
+- `make fmt`: Formats the code
+- `make vet`: Runs go vet
+- `make lint`: Runs golangci-lint
+- `make clean`: Removes build artifacts and installed provider
+
+### Testing
+
+To run the tests:
+```sh
+make test
+```
+
+### Contributing
+
+Contributions are welcome! Please read our [contributing guidelines](CONTRIBUTING.md) first.
+
+## License
+
+This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENSE) file for details.
+
+## Notes
+
+- Always use environment variables or secure secret management solutions to handle sensitive information like API keys and AWS credentials.
+- Refer to the comprehensive documentation in the `docs/` directory for detailed usage examples and configuration options.
+- Make sure to keep your provider version updated for the latest features and bug fixes.
+- The provider now supports AWS cross-account access with `aws_session_name` and `aws_role_name` parameters in the model resource.
+- All example configurations have been consolidated into the documentation for better organization and maintenance.

terraform/provider/RELEASING.md

@@ -0,0 +1,237 @@
+# Release Process
+
+This document describes the release process for the LiteLLM Terraform Provider.
+
+## Overview
+
+Releases are automated via GitHub Actions when a version tag is pushed. The workflow builds the provider for multiple platforms, signs the artifacts with GPG, and publishes them to GitHub Releases.
+
+## Prerequisites
+
+### GPG Key Setup (One-Time Setup for Repository Maintainers)
+
+The Terraform Registry requires all providers to be signed with a GPG key. This must be configured before the first release.
+
+#### 1. Generate a GPG Key
+
+If you don't already have a GPG key for provider signing:
+
+```bash
+gpg --full-generate-key
+```
+
+Configuration:
+- Key type: RSA and RSA (default)
+- Key size: 4096 bits
+- Expiration: No expiration (or set a long expiration period)
+- Email: Use an email associated with your GitHub account
+- Set a strong passphrase (or leave empty for CI/CD use)
+
+#### 2. Export the GPG Key
+
+```bash
+# List your keys to get the key ID
+gpg --list-secret-keys --keyid-format=long
+
+# Example output:
+# sec   rsa4096/ABCD1234EFGH5678 2024-01-01 [SC]
+#       1234567890ABCDEF1234567890ABCDEF12345678
+# uid                 [ultimate] Your Name <your.email@example.com>
+#
+# The key ID is: ABCD1234EFGH5678
+# The fingerprint is: 1234567890ABCDEF1234567890ABCDEF12345678
+
+# Export the private key (ASCII-armored format)
+gpg --armor --export-secret-keys ABCD1234EFGH5678
+
+# Export the public key
+gpg --armor --export ABCD1234EFGH5678
+```
+
+#### 3. Configure GitHub Repository Secrets
+
+Add the following secrets to the repository at: **Settings → Secrets and variables → Actions → New repository secret**
+
+| Secret Name | Description | Value |
+|-------------|-------------|-------|
+| `GPG_PRIVATE_KEY` | The GPG private key for signing releases | Full output from `gpg --armor --export-secret-keys` (including `-----BEGIN PGP PRIVATE KEY BLOCK-----` and `-----END PGP PRIVATE KEY BLOCK-----`) |
+| `PASSPHRASE` | The passphrase for the GPG key | Your GPG key passphrase (leave empty if no passphrase was set) |
+
+#### 4. Register Public Key with Terraform Registry
+
+Before publishing to the Terraform Registry:
+
+1. Go to https://registry.terraform.io/settings/gpg-keys
+2. Click "Add a key"
+3. Paste your public GPG key (output from `gpg --armor --export`)
+4. Submit
+
+**Note**: The public key fingerprint must match the key used to sign the provider releases.
+
+## Release Steps
+
+### 1. Prepare the Release
+
+Before creating a release:
+
+1. **Update CHANGELOG.md**
+   - Move items from `[Unreleased]` section to a new version section
+   - Follow [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format
+   - Use [Semantic Versioning](https://semver.org/spec/v2.0.0.html) for version numbers
+   - Include all notable changes since the last release
+
+   Example:
+   ```markdown
+   ## [0.1.2] - 2026-02-20
+
+   ### Added
+   - New feature description
+
+   ### Fixed
+   - Bug fix description
+
+   ### Changed
+   - Changed behavior description
+   ```
+
+2. **Verify tests pass**
+   ```bash
+   make test
+   ```
+
+3. **Verify the build works locally**
+   ```bash
+   make build
+   ```
+
+4. **Land the changes in BerriAI/litellm**
+
+   Open a PR to `BerriAI/litellm` updating `terraform/provider/CHANGELOG.md` (and any source changes) and merge it. Note the merge commit SHA; the release workflow takes it as `git_ref`
+
+### 2. Mirror and Tag via project-releaser
+
+The provider source lives at `terraform/provider/` in `BerriAI/litellm`; `BerriAI/terraform-provider-litellm` is a thin release mirror. Do not commit or tag the mirror directly
+
+1. Go to `BerriAI/project-releaser` > **Actions** > `Publish Terraform provider`
+2. Click **Run workflow**:
+   - `git_ref`: full 40-char commit SHA from `BerriAI/litellm` to release from
+   - `provider_version`: the new version without the `v` prefix (e.g. `0.3.0`)
+   - `dry_run`: optional; validates without pushing
+3. The workflow rsyncs `terraform/provider/` into the mirror repo, commits, and pushes tag `v<provider_version>`
+4. The tag push triggers the mirror's `Release` workflow (goreleaser), which is gated by the `production-release` environment approval
+
+**Important**:
+- Tags must follow the format: `v<MAJOR>.<MINOR>.<PATCH>` (e.g., `v0.1.2`, `v1.0.0`)
+- The workflow refuses to overwrite an existing tag; publish a new version instead
+
+### 3. Monitor the Release Workflow
+
+1. Go to: https://github.com/BerriAI/terraform-provider-litellm/actions
+2. Find the "Release" workflow run for your tag
+3. Monitor the progress and check for any errors
+
+The workflow will:
+- Check out the code
+- Set up Go
+- Import the GPG key
+- Run `go mod tidy`
+- Build binaries for multiple platforms (Linux, macOS, Windows, FreeBSD)
+- Create archives and checksums
+- Sign the checksums with GPG
+- Create a GitHub release
+- Upload all artifacts
+
+### 4. Verify the Release
+
+After the workflow completes successfully:
+
+1. **Check the GitHub Release**
+   - Go to: https://github.com/BerriAI/terraform-provider-litellm/releases
+   - Verify the release was created with the correct version
+   - Confirm all artifacts are present:
+     - Binary archives for each platform
+     - SHA256SUMS file
+     - SHA256SUMS.sig (GPG signature)
+     - terraform-registry-manifest.json
+
+2. **Verify the signature** (optional)
+   ```bash
+   # Download the checksums and signature
+   wget https://github.com/BerriAI/terraform-provider-litellm/releases/download/v0.1.2/terraform-provider-litellm_0.1.2_SHA256SUMS
+   wget https://github.com/BerriAI/terraform-provider-litellm/releases/download/v0.1.2/terraform-provider-litellm_0.1.2_SHA256SUMS.sig
+
+   # Verify the signature
+   gpg --verify terraform-provider-litellm_0.1.2_SHA256SUMS.sig terraform-provider-litellm_0.1.2_SHA256SUMS
+   ```
+
+### 5. Publish to Terraform Registry (Optional)
+
+If this provider is published to the Terraform Registry:
+
+1. The registry should automatically detect the new release via the GitHub webhook
+2. If not, you may need to manually trigger a sync on the Terraform Registry dashboard
+3. Verify the new version appears at: https://registry.terraform.io/providers/BerriAI/litellm/latest
+
+## Troubleshooting
+
+### Release Workflow Fails with GPG Error
+
+**Error**: `Input required and not supplied: gpg_private_key`
+
+**Solution**:
+- Verify that `GPG_PRIVATE_KEY` and `PASSPHRASE` secrets are configured in the repository
+- Ensure the secrets are not expired
+- Check that the secret names match exactly (case-sensitive)
+
+### GoReleaser Signing Fails
+
+**Error**: `gpg: signing failed: No secret key`
+
+**Solution**:
+- Verify the `GPG_PRIVATE_KEY` secret contains the complete private key block
+- Ensure the passphrase is correct
+- Check that the key hasn't expired: `gpg --list-keys`
+
+### Build Fails
+
+**Error**: Build errors during compilation
+
+**Solution**:
+- Run `make test` and `make build` locally first
+- Ensure `go.mod` and `go.sum` are up to date
+- Check that all dependencies are available
+
+### Tag Already Exists
+
+**Error**: The publish workflow refuses to push because the tag already exists on the mirror
+
+**Solution**: Tags are immutable by design. Re-run the workflow with a new patch version instead of deleting or moving an existing tag
+
+## Version Numbering
+
+This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html):
+
+- **MAJOR** version (1.0.0): Incompatible API changes
+- **MINOR** version (0.1.0): New functionality in a backward-compatible manner
+- **PATCH** version (0.0.1): Backward-compatible bug fixes
+
+For pre-1.0 releases:
+- Breaking changes may occur in minor versions
+- Patch versions should only contain bug fixes
+
+## Security Considerations
+
+1. **Never commit private keys**: The GPG private key should only be stored as a GitHub secret
+2. **Protect repository secrets**: Limit who has access to manage repository secrets
+3. **Use a dedicated key**: Consider using a separate GPG key specifically for provider signing
+4. **Key rotation**: If the GPG key is compromised, generate a new key, update secrets, and register the new public key with the Terraform Registry
+5. **Passphrase**: Use a strong passphrase for the GPG key, or use a passphrase-less key specifically for CI/CD
+
+## References
+
+- [GoReleaser Documentation](https://goreleaser.com/)
+- [Terraform Provider Publishing](https://www.terraform.io/docs/registry/providers/publishing.html)
+- [HashiCorp GPG Signing Requirements](https://www.terraform.io/docs/registry/providers/publishing.html#signing-releases)
+- [GitHub Actions Secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets)
+- [Semantic Versioning](https://semver.org/)
+- [Keep a Changelog](https://keepachangelog.com/)

terraform/provider/docs/data-sources/credential.md

@@ -0,0 +1,153 @@
+---
+# generated by https://github.com/hashicorp/terraform-plugin-docs
+page_title: "litellm_credential Data Source - terraform-provider-litellm"
+subcategory: ""
+description: |-
+  Retrieves information about an existing LiteLLM credential.
+---
+
+# litellm_credential (Data Source)
+
+Retrieves information about an existing LiteLLM credential. This data source allows you to reference credentials that were created outside of Terraform or in other Terraform configurations.
+
+## Example Usage
+
+```terraform
+# Retrieve an existing credential by name
+data "litellm_credential" "existing_openai" {
+  credential_name = "openai-production-key"
+}
+
+# Use the credential in a model resource
+resource "litellm_model" "gpt4_with_existing_cred" {
+  model_name          = "gpt-4-with-existing-cred"
+  custom_llm_provider = "openai"
+  base_model          = "gpt-4"
+  tier                = "paid"
+  mode                = "chat"
+  
+  # Reference the existing credential's info
+  additional_litellm_params = {
+    credential_name = data.litellm_credential.existing_openai.credential_name
+  }
+}
+```
+
+## Example Usage with Model ID
+
+```terraform
+# Retrieve a credential associated with a specific model
+data "litellm_credential" "model_specific_cred" {
+  credential_name = "claude-api-key"
+  model_id        = "claude-3-sonnet"
+}
+
+# Use in a vector store
+resource "litellm_vector_store" "knowledge_base" {
+  vector_store_name       = "claude-knowledge-base"
+  custom_llm_provider     = "anthropic"
+  litellm_credential_name = data.litellm_credential.model_specific_cred.credential_name
+  
+  vector_store_description = "Knowledge base using Claude credentials"
+}
+```
+
+## Example Usage for Cross-Reference
+
+```terraform
+# Get credential info to use in other resources
+data "litellm_credential" "shared_cred" {
+  credential_name = "shared-api-key"
+}
+
+# Create multiple resources using the same credential
+resource "litellm_vector_store" "store_1" {
+  vector_store_name       = "store-1"
+  custom_llm_provider     = "pinecone"
+  litellm_credential_name = data.litellm_credential.shared_cred.credential_name
+  
+  vector_store_description = "First store using shared credential"
+}
+
+resource "litellm_vector_store" "store_2" {
+  vector_store_name       = "store-2"
+  custom_llm_provider     = "pinecone"
+  litellm_credential_name = data.litellm_credential.shared_cred.credential_name
+  
+  vector_store_description = "Second store using shared credential"
+}
+```
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `credential_name` - (Required) Name of the credential to retrieve.
+* `model_id` - (Optional) Model ID associated with this credential. Use this when the same credential name is used for different models.
+
+## Attributes Reference
+
+In addition to all arguments above, the following attributes are exported:
+
+* `credential_info` - Map of additional non-sensitive information about the credential.
+
+## Security Note
+
+For security reasons, the `credential_values` (sensitive data like API keys) are not exposed through data sources. This prevents accidental exposure of sensitive information in Terraform plans and logs. If you need to access credential values, you should manage them through the resource directly or use external secret management systems.
+
+## Common Use Cases
+
+### 1. Cross-Stack References
+Use data sources to reference credentials created in other Terraform configurations or stacks:
+
+```terraform
+data "litellm_credential" "shared_openai" {
+  credential_name = "openai-shared-key"
+}
+
+resource "litellm_model" "gpt4" {
+  model_name          = "gpt-4-cross-stack"
+  custom_llm_provider = "openai"
+  base_model          = "gpt-4"
+  
+  additional_litellm_params = {
+    credential_reference = data.litellm_credential.shared_openai.credential_name
+  }
+}
+```
+
+### 2. Conditional Logic
+Use credential information for conditional resource creation:
+
+```terraform
+data "litellm_credential" "optional_cred" {
+  credential_name = var.credential_name
+}
+
+resource "litellm_vector_store" "conditional_store" {
+  count = length(data.litellm_credential.optional_cred.credential_info) > 0 ? 1 : 0
+  
+  vector_store_name       = "conditional-store"
+  custom_llm_provider     = "weaviate"
+  litellm_credential_name = data.litellm_credential.optional_cred.credential_name
+}
+```
+
+### 3. Validation and Verification
+Verify that required credentials exist before creating dependent resources:
+
+```terraform
+data "litellm_credential" "required_cred" {
+  credential_name = "production-api-key"
+}
+
+# This will fail if the credential doesn't exist
+resource "litellm_model" "production_model" {
+  model_name          = "production-gpt-4"
+  custom_llm_provider = "openai"
+  base_model          = "gpt-4"
+  
+  additional_litellm_params = {
+    credential_name = data.litellm_credential.required_cred.credential_name
+  }
+}

terraform/provider/docs/data-sources/vector_store.md

@@ -0,0 +1,225 @@
+---
+# generated by https://github.com/hashicorp/terraform-plugin-docs
+page_title: "litellm_vector_store Data Source - terraform-provider-litellm"
+subcategory: ""
+description: |-
+  Retrieves information about an existing LiteLLM vector store.
+---
+
+# litellm_vector_store (Data Source)
+
+Retrieves information about an existing LiteLLM vector store. This data source allows you to reference vector stores that were created outside of Terraform or in other Terraform configurations.
+
+## Example Usage
+
+```terraform
+# Retrieve an existing vector store by ID
+data "litellm_vector_store" "existing_store" {
+  vector_store_id = "vs-12345"
+}
+
+# Use the vector store information in outputs
+output "vector_store_info" {
+  value = {
+    name        = data.litellm_vector_store.existing_store.vector_store_name
+    provider    = data.litellm_vector_store.existing_store.custom_llm_provider
+    created_at  = data.litellm_vector_store.existing_store.created_at
+  }
+}
+```
+
+## Example Usage for Cross-Reference
+
+```terraform
+# Get vector store info to reference in other configurations
+data "litellm_vector_store" "shared_store" {
+  vector_store_id = var.shared_vector_store_id
+}
+
+# Create a model that might use the same credential as the vector store
+data "litellm_credential" "store_credential" {
+  credential_name = data.litellm_vector_store.shared_store.litellm_credential_name
+}
+
+resource "litellm_model" "embedding_model" {
+  model_name          = "embedding-model"
+  custom_llm_provider = "openai"
+  base_model          = "text-embedding-ada-002"
+  mode                = "embedding"
+  
+  additional_litellm_params = {
+    credential_name = data.litellm_credential.store_credential.credential_name
+  }
+}
+```
+
+## Example Usage for Validation
+
+```terraform
+# Verify vector store exists and get its configuration
+data "litellm_vector_store" "production_store" {
+  vector_store_id = "production-vector-store-id"
+}
+
+# Create resources only if the vector store is properly configured
+resource "litellm_model" "rag_model" {
+  count = data.litellm_vector_store.production_store.custom_llm_provider == "pinecone" ? 1 : 0
+  
+  model_name          = "rag-enabled-model"
+  custom_llm_provider = "openai"
+  base_model          = "gpt-4"
+  mode                = "chat"
+  
+  additional_litellm_params = {
+    vector_store_id = data.litellm_vector_store.production_store.vector_store_id
+  }
+}
+```
+
+## Example Usage for Monitoring
+
+```terraform
+# Get vector store details for monitoring and alerting
+data "litellm_vector_store" "monitored_stores" {
+  for_each = toset(var.vector_store_ids)
+  
+  vector_store_id = each.value
+}
+
+# Output store information for monitoring systems
+output "vector_store_status" {
+  value = {
+    for k, v in data.litellm_vector_store.monitored_stores : k => {
+      name         = v.vector_store_name
+      provider     = v.custom_llm_provider
+      created_at   = v.created_at
+      updated_at   = v.updated_at
+      metadata     = v.vector_store_metadata
+    }
+  }
+}
+```
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `vector_store_id` - (Required) Unique identifier for the vector store to retrieve.
+
+## Attributes Reference
+
+In addition to all arguments above, the following attributes are exported:
+
+* `vector_store_name` - Name of the vector store.
+* `custom_llm_provider` - Custom LLM provider for the vector store.
+* `vector_store_description` - Description of the vector store.
+* `vector_store_metadata` - Map of metadata associated with the vector store.
+* `litellm_credential_name` - Name of the LiteLLM credential used.
+* `litellm_params` - Map of additional LiteLLM parameters.
+* `created_at` - Timestamp when the vector store was created.
+* `updated_at` - Timestamp when the vector store was last updated.
+
+## Common Use Cases
+
+### 1. Cross-Stack References
+Reference vector stores created in other Terraform configurations:
+
+```terraform
+data "litellm_vector_store" "shared_knowledge_base" {
+  vector_store_id = var.knowledge_base_id
+}
+
+# Use the same credential for consistency
+resource "litellm_model" "knowledge_model" {
+  model_name          = "knowledge-retrieval-model"
+  custom_llm_provider = "openai"
+  base_model          = "gpt-4"
+  
+  additional_litellm_params = {
+    vector_store_credential = data.litellm_vector_store.shared_knowledge_base.litellm_credential_name
+  }
+}
+```
+
+### 2. Configuration Validation
+Validate vector store configuration before creating dependent resources:
+
+```terraform
+data "litellm_vector_store" "target_store" {
+  vector_store_id = var.target_vector_store_id
+}
+
+# Ensure the vector store uses the expected provider
+locals {
+  is_pinecone_store = data.litellm_vector_store.target_store.custom_llm_provider == "pinecone"
+}
+
+resource "litellm_model" "pinecone_optimized_model" {
+  count = local.is_pinecone_store ? 1 : 0
+  
+  model_name          = "pinecone-optimized"
+  custom_llm_provider = "openai"
+  base_model          = "text-embedding-ada-002"
+  mode                = "embedding"
+}
+```
+
+### 3. Metadata-Based Logic
+Use vector store metadata for conditional resource creation:
+
+```terraform
+data "litellm_vector_store" "environment_store" {
+  vector_store_id = var.vector_store_id
+}
+
+# Create different resources based on environment metadata
+resource "litellm_model" "production_model" {
+  count = lookup(data.litellm_vector_store.environment_store.vector_store_metadata, "environment", "") == "production" ? 1 : 0
+  
+  model_name          = "production-rag-model"
+  custom_llm_provider = "openai"
+  base_model          = "gpt-4"
+  mode                = "chat"
+}
+
+resource "litellm_model" "development_model" {
+  count = lookup(data.litellm_vector_store.environment_store.vector_store_metadata, "environment", "") == "development" ? 1 : 0
+  
+  model_name          = "development-rag-model"
+  custom_llm_provider = "openai"
+  base_model          = "gpt-3.5-turbo"
+  mode                = "chat"
+}
+```
+
+### 4. Audit and Compliance
+Retrieve vector store information for audit and compliance reporting:
+
+```terraform
+data "litellm_vector_store" "compliance_stores" {
+  for_each = toset(var.compliance_vector_store_ids)
+  
+  vector_store_id = each.value
+}
+
+# Generate compliance report
+output "compliance_report" {
+  value = {
+    for k, v in data.litellm_vector_store.compliance_stores : k => {
+      store_name   = v.vector_store_name
+      provider     = v.custom_llm_provider
+      credential   = v.litellm_credential_name
+      created_date = v.created_at
+      last_updated = v.updated_at
+      metadata     = v.vector_store_metadata
+    }
+  }
+}
+```
+
+## Notes
+
+* Vector store IDs are unique identifiers assigned by the LiteLLM system.
+* The data source will fail if the specified vector store ID does not exist.
+* All computed attributes reflect the current state of the vector store in the LiteLLM system.
+* Use this data source to integrate with existing vector stores or to reference stores created outside of Terraform.