Merge branch 'litellm_internal_staging' into feature/improve-gigachat-provider

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -30,7 +30,7 @@ body:
     id: steps-to-reproduce
     attributes:
       label: Steps to Reproduce
-      description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
+      description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them.
       placeholder: |
         1. config.yaml file/ .env file/ etc.
         2. Run the following code...

.github/pull_request_template.md

@@ -1,3 +1,18 @@
+## TLDR
+
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
+     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+
+Problem this solves:
+
+- <blah>
+- ...
+
+How it solves it:
+
+- <blah>
+- ...
+
 ## Relevant issues
 
 <!-- e.g., "Fixes #000" -->

.github/workflows/image-scan.yml

@@ -58,6 +58,8 @@ jobs:
       # free OSS, run as a pinned, checksum-verified binary; no GitHub Action
       # dependency and no vendor SaaS callout.
       - name: Scan image for fixable HIGH/CRITICAL CVEs
+        env:
+          GRYPE_MATCH_PYTHON_USING_CPES: "true"
         run: |
           "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \
             --only-fixed \

.github/workflows/test-code-quality.yml

@@ -115,6 +115,9 @@ jobs:
       - name: check_fastuuid_usage
         run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
 
+      - name: check_e2e_no_raw_requests
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-litellm-ui-unit.yml

@@ -0,0 +1,57 @@
+name: UI Unit Tests
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - litellm_internal_staging
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ui-unit-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    defaults:
+      run:
+        working-directory: ui/litellm-dashboard
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run UI unit tests (Vitest)
+        env:
+          CI: "true"
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          if [ -n "$BASE_SHA" ]; then
+            echo "Pull request: running only tests related to changes since $BASE_SHA"
+            npm run test -- --run --changed "$BASE_SHA" --passWithNoTests \
+              --pool forks --poolOptions.forks.maxForks=4
+          else
+            echo "Push to $GITHUB_REF_NAME: running the full suite"
+            npm run test -- --run --pool forks --poolOptions.forks.maxForks=4
+          fi

.github/workflows/weekly_load_anomaly.yml

@@ -0,0 +1,81 @@
+name: "Weekly Load Anomaly Check"
+
+on:
+  schedule:
+    - cron: "0 12 * * 6"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  weekly-load-anomaly:
+    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-weekly-anomaly-check
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/load/weekly_anomaly_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Run the weekly session anomaly test
+        env:
+          E2E_WEEKLY_ANOMALY: "1"
+        run: |
+          uv run --no-sync pytest tests/e2e/load/test_weekly_session_anomaly_e2e.py -v --tb=short -rA
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

litellm-proxy-extras/litellm_proxy_extras/migrations/20260721000000_add_sso_identity_assertion/migration.sql

@@ -0,0 +1,9 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_SSOIdentityAssertion" (
+    "user_id" TEXT NOT NULL,
+    "assertion_b64" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_SSOIdentityAssertion_pkey" PRIMARY KEY ("user_id")
+);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -403,6 +403,15 @@ model LiteLLM_MCPServerOAuthClient {
   updated_at  DateTime @default(now()) @updatedAt @map("updated_at")
 }
 
+// The enterprise IdP identity assertion captured at SSO login, one row per user.
+// assertion_b64 is an encrypted JSON payload: {id_token, refresh_token?, issuer?, expires_at?}.
+model LiteLLM_SSOIdentityAssertion {
+  user_id       String   @id
+  assertion_b64 String
+  created_at    DateTime @default(now()) @map("created_at")
+  updated_at    DateTime @default(now()) @updatedAt @map("updated_at")
+}
+
 // Generate Tokens for Proxy
 model LiteLLM_VerificationToken {
     token      String   @id

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.79"
+version = "0.4.80"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.79"
+version = "0.4.80"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/ai-gateway/src/messages/common_utils.rs

@@ -50,3 +50,15 @@ pub(super) fn has_header(headers: &[(String, String)], name: &str) -> bool {
         .iter()
         .any(|(key, _)| key.eq_ignore_ascii_case(name))
 }
+
+pub(super) fn has_bearer_auth(headers: &[(String, String)]) -> bool {
+    headers.iter().any(|(name, value)| {
+        if !name.eq_ignore_ascii_case("authorization") {
+            return false;
+        }
+        let value = value.trim();
+        value.len() > 7
+            && value[..7].eq_ignore_ascii_case("bearer ")
+            && !value[7..].trim().is_empty()
+    })
+}

litellm-rust/crates/ai-gateway/src/messages/prepare.rs

@@ -3,7 +3,7 @@ use litellm_core::CoreResult;
 use litellm_core::messages::transformation::MessagesAuthStrategy;
 use litellm_core::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
 
-use super::common_utils::{has_header, messages_provider_config, string_headers};
+use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers};
 use super::types::{MessagesRequest, ProviderMessagesRequest};
 
 pub(super) fn prepare_messages_call(
@@ -33,7 +33,9 @@ pub(super) fn prepare_messages_call(
     let mut headers = string_headers(request.extra_headers)?;
 
     let auth_strategy = config.auth_strategy();
-    if !has_header(&headers, auth_strategy.header_name()) {
+    let already_authorized = has_header(&headers, auth_strategy.header_name())
+        || (config.accepts_bearer_auth() && has_bearer_auth(&headers));
+    if !already_authorized {
         let api_key = config.resolve_api_key(request.api_key, &env_lookup)?;
         let auth_header = match auth_strategy {
             MessagesAuthStrategy::Bearer => {

litellm-rust/crates/ai-gateway/src/messages/tests.rs

@@ -6,7 +6,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::net::{TcpListener, TcpStream};
 
 use super::common_utils::{
-    has_header, messages_provider_config, string_headers, truncate_error_body,
+    has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body,
 };
 use super::{MessagesRequest, messages};
 
@@ -85,6 +85,34 @@ fn has_header_is_case_insensitive() {
     assert!(!has_header(&headers, "authorization"));
 }
 
+#[test]
+fn has_bearer_auth_requires_a_nonempty_bearer_token() {
+    assert!(has_bearer_auth(&[(
+        "Authorization".to_string(),
+        "Bearer tok".to_string()
+    )]));
+    assert!(has_bearer_auth(&[(
+        "authorization".to_string(),
+        "bearer tok".to_string()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "authorization".to_string(),
+        "Bearer ".to_string()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "authorization".to_string(),
+        String::new()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "authorization".to_string(),
+        "Basic abc".to_string()
+    )]));
+    assert!(!has_bearer_auth(&[(
+        "x-api-key".to_string(),
+        "sk".to_string()
+    )]));
+}
+
 #[tokio::test]
 async fn messages_round_trip_builds_azure_request_and_passes_response_through() {
     let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
@@ -252,6 +280,112 @@ async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() {
     assert!(!head.contains("rust-fallback-key"), "{head}");
 }
 
+#[tokio::test]
+async fn messages_forwards_entra_id_bearer_without_requiring_api_key() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body =
+            r#"{"id":"msg_3","type":"message","role":"assistant","content":[],"model":"m"}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let mut headers = Map::new();
+    headers.insert(
+        "Authorization".to_string(),
+        Value::String("Bearer entra-token".to_string()),
+    );
+
+    messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: None,
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: Some(headers),
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("entra id request succeeds without api key");
+
+    let request = server.await.expect("server task completes");
+    let head = request
+        .split_once("\r\n\r\n")
+        .expect("has body")
+        .0
+        .to_ascii_lowercase();
+    assert!(head.contains("authorization: bearer entra-token"), "{head}");
+    assert!(!head.contains("x-api-key"), "{head}");
+}
+
+#[tokio::test]
+async fn messages_requires_auth_when_no_key_and_no_header() {
+    let err = messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: None,
+        api_base: Some("http://127.0.0.1:1"),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: None,
+        timeout: Some(Duration::from_millis(50)),
+    })
+    .await
+    .expect_err("missing auth errors");
+
+    assert!(matches!(err, CoreError::Auth(_)));
+}
+
+#[tokio::test]
+async fn messages_ignores_malformed_authorization_and_uses_api_key() {
+    let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");
+    let addr = listener.local_addr().expect("addr");
+
+    let server = tokio::spawn(async move {
+        let (mut socket, _) = listener.accept().await.expect("accepts request");
+        let request = read_http_request(&mut socket).await;
+        let response_body =
+            r#"{"id":"msg_4","type":"message","role":"assistant","content":[],"model":"m"}"#;
+        socket
+            .write_all(write_response(response_body).as_bytes())
+            .await
+            .expect("writes response");
+        request
+    });
+
+    let mut headers = Map::new();
+    headers.insert(
+        "Authorization".to_string(),
+        Value::String("Bearer ".to_string()),
+    );
+
+    messages(MessagesRequest {
+        model: "claude-sonnet-4-5",
+        body: json!({"model": "claude-sonnet-4-5", "max_tokens": 8, "messages": []}),
+        api_key: Some("sk-azure"),
+        api_base: Some(&format!("http://{addr}")),
+        custom_llm_provider: Some("azure_ai"),
+        extra_headers: Some(headers),
+        timeout: Some(Duration::from_secs(5)),
+    })
+    .await
+    .expect("falls back to api key");
+
+    let request = server.await.expect("server task completes");
+    let head = request
+        .split_once("\r\n\r\n")
+        .expect("has body")
+        .0
+        .to_ascii_lowercase();
+    assert!(head.contains("x-api-key: sk-azure"), "{head}");
+}
+
 #[tokio::test]
 async fn messages_maps_provider_error_status_to_http_error() {
     let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds");

litellm-rust/crates/core/src/messages/transformation.rs

@@ -35,6 +35,10 @@ pub trait AnthropicMessagesProviderConfig: Sync {
         MessagesAuthStrategy::Header("x-api-key")
     }
 
+    fn accepts_bearer_auth(&self) -> bool {
+        false
+    }
+
     fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
         &[
             ("anthropic-version", "2023-06-01"),

litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs

@@ -163,6 +163,10 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig {
         self.anthropic.auth_strategy()
     }
 
+    fn accepts_bearer_auth(&self) -> bool {
+        true
+    }
+
     fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
         self.anthropic.default_headers()
     }
@@ -294,6 +298,11 @@ mod tests {
         );
     }
 
+    #[test]
+    fn accepts_bearer_auth_for_entra_id() {
+        assert!(AZURE_ANTHROPIC_MESSAGES_CONFIG.accepts_bearer_auth());
+    }
+
     #[test]
     fn default_headers_match_python() {
         assert_eq!(

litellm/constants.py

@@ -1469,6 +1469,7 @@ _batch_polling_env = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower()
 PROXY_BATCH_POLLING_ENABLED = _batch_polling_env == "true"
 PROXY_BUDGET_RESCHEDULER_MAX_TIME = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605))
 PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10))  # in seconds, increased from 10
+PROXY_CONFIG_RELOAD_INTERVAL_SECONDS = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)
 
 # APScheduler Configuration - MEMORY LEAK FIX
 # These settings prevent memory leaks in APScheduler's normalize() and _apply_jitter() functions

litellm/litellm_core_utils/duration_parser.py

@@ -7,8 +7,8 @@ duration_in_seconds is used in diff parts of the code base, example
 """
 
 import re
-import time
-from datetime import datetime, timedelta, timezone, tzinfo
+import time as time_module
+from datetime import datetime, time, timedelta, timezone, tzinfo
 from typing import Optional, Tuple
 from zoneinfo import ZoneInfo
 
@@ -61,7 +61,7 @@ def duration_in_seconds(duration: str) -> int:
     elif unit == "w":
         return value * 604800
     elif unit == "mo":
-        now = time.time()
+        now = time_module.time()
         current_time = datetime.fromtimestamp(now)
 
         # Calculate target month and year, handling overflow past December
@@ -94,12 +94,17 @@ def duration_in_seconds(duration: str) -> int:
         raise ValueError(f"Unsupported duration unit, passed duration: {duration}")
 
 
-def get_next_standardized_reset_time(duration: str, current_time: datetime, timezone_str: str = "UTC") -> datetime:
+def get_next_standardized_reset_time(
+    duration: str,
+    current_time: datetime,
+    timezone_str: str = "UTC",
+    reset_time_of_day: time = time(0, 0),
+) -> datetime:
     """
     Get the next standardized reset time based on the duration.
 
     All durations will reset at predictable intervals, aligned from the current time:
-    - Nd: If N=1, reset at next midnight; if N>1, reset every N days from now
+    - Nd: If N=1, reset at the next `reset_time_of_day`; if N>1, reset every N days from now
     - Nh: Every N hours, aligned to hour boundaries (e.g., 1:00, 2:00)
     - Nm: Every N minutes, aligned to minute boundaries (e.g., 1:05, 1:10)
     - Ns: Every N seconds, aligned to second boundaries
@@ -108,12 +113,15 @@ def get_next_standardized_reset_time(duration: str, current_time: datetime, time
     - duration: Duration string (e.g. "30s", "30m", "30h", "30d")
     - current_time: Current datetime
     - timezone_str: Timezone string (e.g. "UTC", "US/Eastern", "Asia/Kolkata")
+    - reset_time_of_day: Wall-clock time the reset lands on for day/week/month
+      durations (defaults to midnight). Ignored for sub-day durations, where a
+      time-of-day is meaningless.
 
     Returns:
     - Next reset time at a standardized interval in the specified timezone
     """
     # Set up timezone and normalize current time
-    current_time, tz = _setup_timezone(current_time, timezone_str)
+    current_time, _ = _setup_timezone(current_time, timezone_str)
 
     # Parse duration
     value, unit = _parse_duration(duration)
@@ -126,9 +134,9 @@ def get_next_standardized_reset_time(duration: str, current_time: datetime, time
 
     # Handle different time units
     if unit == "d":
-        return _handle_day_reset(current_time, base_midnight, value, tz)
+        return _handle_day_reset(current_time, base_midnight, value, reset_time_of_day)
     elif unit == "w":
-        return _handle_day_reset(current_time, base_midnight, value * 7, tz)
+        return _handle_day_reset(current_time, base_midnight, value * 7, reset_time_of_day)
     elif unit == "h":
         return _handle_hour_reset(current_time, base_midnight, value)
     elif unit == "m":
@@ -136,7 +144,7 @@ def get_next_standardized_reset_time(duration: str, current_time: datetime, time
     elif unit == "s":
         return _handle_second_reset(current_time, base_midnight, value)
     elif unit == "mo":
-        return _handle_month_reset(current_time, base_midnight, value)
+        return _handle_month_reset(current_time, base_midnight, value, reset_time_of_day)
     else:
         # Unrecognized unit, default to next midnight
         return base_midnight + timedelta(days=1)
@@ -175,46 +183,58 @@ def _parse_duration(duration: str) -> Tuple[Optional[int], Optional[str]]:
     return int(value), unit
 
 
-def _handle_day_reset(current_time: datetime, base_midnight: datetime, value: int, tz: tzinfo) -> datetime:
+def _apply_time_of_day(dt: datetime, reset_time_of_day: time) -> datetime:
+    """Set the wall-clock time of `dt` to `reset_time_of_day`, keeping its date and tzinfo."""
+    return dt.replace(
+        hour=reset_time_of_day.hour,
+        minute=reset_time_of_day.minute,
+        second=reset_time_of_day.second,
+        microsecond=reset_time_of_day.microsecond,
+    )
+
+
+def _next_occurrence(
+    boundary_midnight: datetime,
+    reset_time_of_day: time,
+    current_time: datetime,
+    period: timedelta,
+) -> datetime:
+    """Place the reset at `reset_time_of_day` on the boundary day, rolling forward one
+    `period` if that instant has already passed (or is exactly now)."""
+    candidate = _apply_time_of_day(boundary_midnight, reset_time_of_day)
+    if candidate <= current_time:
+        return candidate + period
+    return candidate
+
+
+def _first_of_next_month(first_of_month: datetime) -> datetime:
+    """Given the 1st of some month, return the 1st of the following month."""
+    if first_of_month.month == 12:
+        return first_of_month.replace(year=first_of_month.year + 1, month=1)
+    return first_of_month.replace(month=first_of_month.month + 1)
+
+
+def _handle_day_reset(
+    current_time: datetime,
+    base_midnight: datetime,
+    value: int,
+    reset_time_of_day: time,
+) -> datetime:
     """Handle day-based reset times."""
     # Handle zero value - immediate expiration
     if value == 0:
         return current_time
 
-    if value == 1:  # Daily reset at midnight
-        return base_midnight + timedelta(days=1)
-    elif value == 7:  # Weekly reset on Monday at midnight
+    if value == 1:  # Daily reset at the configured time of day
+        return _next_occurrence(base_midnight, reset_time_of_day, current_time, timedelta(days=1))
+    elif value == 7:  # Weekly reset on Monday at the configured time of day
         days_until_monday = (7 - current_time.weekday()) % 7
-        if days_until_monday == 0:  # If today is Monday
-            days_until_monday = 7
-        return base_midnight + timedelta(days=days_until_monday)
-    elif value == 30:  # Monthly reset on 1st at midnight
-        # Get 1st of next month at midnight
-        if current_time.month == 12:
-            next_reset = datetime(
-                year=current_time.year + 1,
-                month=1,
-                day=1,
-                hour=0,
-                minute=0,
-                second=0,
-                microsecond=0,
-                tzinfo=tz,
-            )
-        else:
-            next_reset = datetime(
-                year=current_time.year,
-                month=current_time.month + 1,
-                day=1,
-                hour=0,
-                minute=0,
-                second=0,
-                microsecond=0,
-                tzinfo=tz,
-            )
-        return next_reset
-    else:  # Custom day value - next interval is value days from current
-        return current_time.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta(days=value)
+        upcoming_monday = base_midnight + timedelta(days=days_until_monday)
+        return _next_occurrence(upcoming_monday, reset_time_of_day, current_time, timedelta(days=7))
+    elif value == 30:  # Monthly reset on 1st at the configured time of day
+        return _handle_month_reset(current_time, base_midnight, 1, reset_time_of_day)
+    else:  # Custom day value - next interval is value days from the start of today
+        return _apply_time_of_day(base_midnight + timedelta(days=value), reset_time_of_day)
 
 
 def _handle_hour_reset(current_time: datetime, base_midnight: datetime, value: int) -> datetime:
@@ -316,36 +336,30 @@ def _handle_second_reset(current_time: datetime, base_midnight: datetime, value:
     return current_time.replace(hour=next_hour, minute=next_minute, second=next_second, microsecond=0)
 
 
-def _handle_month_reset(current_time: datetime, base_midnight: datetime, value: int) -> datetime:
+def _handle_month_reset(
+    current_time: datetime,
+    base_midnight: datetime,
+    value: int,
+    reset_time_of_day: time,
+) -> datetime:
     """
-    Handle monthly reset times. For monthly resets, we always reset at the start of the next month.
+    Handle monthly reset times. Resets land on the 1st at `reset_time_of_day`; if the
+    1st of the current month at that time has already passed, roll to the 1st of next month.
 
     Args:
         current_time: Current datetime
         base_midnight: Midnight of current day
         value: Number of months (currently only supports 1 month resets)
+        reset_time_of_day: Wall-clock time the reset lands on
 
     Returns:
-        datetime: First day of next month at midnight
+        datetime: First day of the next reset month at `reset_time_of_day`
     """
     if value != 1:
         raise ValueError("Monthly resets currently only support 1 month intervals")
 
-    # Get the first day of next month
-    if current_time.month == 12:
-        next_month = 1
-        next_year = current_time.year + 1
-    else:
-        next_month = current_time.month + 1
-        next_year = current_time.year
-
-    return datetime(
-        year=next_year,
-        month=next_month,
-        day=1,
-        hour=0,
-        minute=0,
-        second=0,
-        microsecond=0,
-        tzinfo=current_time.tzinfo,
-    )
+    first_of_this_month = base_midnight.replace(day=1)
+    candidate = _apply_time_of_day(first_of_this_month, reset_time_of_day)
+    if candidate <= current_time:
+        return _apply_time_of_day(_first_of_next_month(first_of_this_month), reset_time_of_day)
+    return candidate

litellm/litellm_core_utils/url_utils.py

@@ -148,6 +148,15 @@ def _parse_url_destination_allowlist_entry(
     return _normalize_host(parsed.hostname), scheme, port
 
 
+def provider_url_destination_candidates(value: str) -> Tuple[str, ...]:
+    return tuple(
+        candidate
+        for part in value.split(",")
+        for candidate in (part.strip(), part.strip().split("/", 1)[1] if "/" in part.strip() else "")
+        if candidate
+    )
+
+
 def is_url_destination_allowed_by_host(url: str, allowed_hosts: List[str]) -> bool:
     """Return True when a credential-bearing provider URL is admin-allowlisted.
 

litellm/llms/bedrock/realtime/handler.py

@@ -9,6 +9,8 @@ import contextlib
 import json
 from typing import Any, Optional
 
+from pydantic import TypeAdapter
+
 from litellm._logging import _redact_string, verbose_proxy_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 
@@ -16,6 +18,8 @@ from ..base_aws_llm import BaseAWSLLM
 from ..common_utils import BedrockError
 from .transformation import BedrockRealtimeConfig
 
+_CLIENT_MODALITIES_ADAPTER: TypeAdapter["list[str] | None"] = TypeAdapter(list[str] | None)
+
 
 class BedrockRealtime(BaseAWSLLM):
     """Handler for Bedrock Nova Sonic realtime speech-to-speech API."""
@@ -124,6 +128,9 @@ class BedrockRealtime(BaseAWSLLM):
 
             verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established")
 
+            await websocket.send_text(json.dumps(transformation_config.session_created_event(model, logging_obj)))
+            verbose_proxy_logger.debug("Bedrock Realtime: sent session.created to client on connect")
+
             # Track state for transformation
             session_state = {
                 "current_output_item_id": None,
@@ -143,6 +150,7 @@ class BedrockRealtime(BaseAWSLLM):
                     transformation_config,
                     model,
                     session_state,
+                    logging_obj,
                 )
             )
 
@@ -179,6 +187,7 @@ class BedrockRealtime(BaseAWSLLM):
         transformation_config: BedrockRealtimeConfig,
         model: str,
         session_state: dict,
+        logging_obj: LiteLLMLogging | None = None,
     ):
         """Forward messages from client WebSocket to Bedrock stream."""
         from aws_sdk_bedrock_runtime.models import (
@@ -210,6 +219,23 @@ class BedrockRealtime(BaseAWSLLM):
                 for bedrock_message in transformed_messages:
                     await send_to_bedrock(bedrock_message)
 
+                if logging_obj is not None:
+                    client_message_type: str | None = None
+                    requested_modalities: list[str] | None = None
+                    with contextlib.suppress(Exception):
+                        parsed_client_message = json.loads(message)
+                        client_message_type = parsed_client_message.get("type")
+                        if client_message_type == "session.update":
+                            requested_modalities = _CLIENT_MODALITIES_ADAPTER.validate_python(
+                                parsed_client_message.get("session", {}).get("modalities")
+                            )
+                    if client_message_type == "session.update":
+                        await client_ws.send_text(
+                            json.dumps(
+                                transformation_config.session_updated_event(model, logging_obj, requested_modalities)
+                            )
+                        )
+
         except Exception as e:
             verbose_proxy_logger.debug(f"Client to Bedrock forwarding ended: {e}", exc_info=True)
             for close_message in transformation_config.session_close_messages():

litellm/llms/bedrock/realtime/transformation.py

@@ -623,35 +623,42 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             verbose_logger.warning(f"Unknown message type: {message_type}")
             return []
 
-    def transform_session_start_event(
+    def _session_object(
         self,
-        event: dict,
         model: str,
         logging_obj: LiteLLMLoggingObj,
-    ) -> OpenAIRealtimeStreamSessionEvents:
-        """
-        Transform Bedrock sessionStart event to OpenAI session.created.
-
-        Args:
-            event: Bedrock sessionStart event
-            model: Model ID
-            logging_obj: Logging object
-
-        Returns:
-            OpenAI session.created event
-        """
-        verbose_logger.debug("Handling sessionStart")
-
+        modalities: list[str] | None = None,
+    ) -> OpenAIRealtimeStreamSession:
         session = OpenAIRealtimeStreamSession(
             id=logging_obj.litellm_trace_id,
-            modalities=["text", "audio"],
+            modalities=modalities if modalities is not None else ["text", "audio"],
         )
         if model is not None and isinstance(model, str):
             session["model"] = model
+        return session
 
+    def session_created_event(
+        self,
+        model: str,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> OpenAIRealtimeStreamSessionEvents:
+        """Build the OpenAI session.created event for this realtime session."""
         return OpenAIRealtimeStreamSessionEvents(
             type="session.created",
-            session=session,
+            session=self._session_object(model, logging_obj),
+            event_id=str(uuid.uuid4()),
+        )
+
+    def session_updated_event(
+        self,
+        model: str,
+        logging_obj: LiteLLMLoggingObj,
+        modalities: list[str] | None = None,
+    ) -> OpenAIRealtimeStreamSessionEvents:
+        """Build the OpenAI session.updated ack reflecting the client's requested modalities."""
+        return OpenAIRealtimeStreamSessionEvents(
+            type="session.updated",
+            session=self._session_object(model, logging_obj, modalities),
             event_id=str(uuid.uuid4()),
         )
 
@@ -1169,8 +1176,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
 
         # Route to appropriate transformation method
         if "sessionStart" in event:
-            session_created = self.transform_session_start_event(event, model, logging_obj)
-            returned_messages.append(session_created)
             session_configuration_request = json.dumps({"configured": True})
 
         elif "contentStart" in event:

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2107,8 +2107,7 @@ class BaseLLMHTTPHandler:
         rust_messages_response = await self._maybe_rust_anthropic_messages(
             custom_llm_provider=custom_llm_provider,
             litellm_params=litellm_params,
-            stream=stream or False,
-            rust_stream_eligible=bool(stream) and not self._has_agentic_completion_hook(logging_obj),
+            has_agentic_hook=self._has_agentic_completion_hook(logging_obj),
             model=model,
             api_key=api_key,
             api_base=api_base,
@@ -2266,8 +2265,7 @@ class BaseLLMHTTPHandler:
         *,
         custom_llm_provider: str,
         litellm_params: GenericLiteLLMParams,
-        stream: bool,
-        rust_stream_eligible: bool,
+        has_agentic_hook: bool,
         model: str,
         api_key: str | None,
         api_base: str | None,
@@ -2279,7 +2277,7 @@ class BaseLLMHTTPHandler:
             return None
         if litellm_params.get("rust") is not True and not BaseLLMHTTPHandler._rust_env_enabled():
             return None
-        if stream and not rust_stream_eligible:
+        if has_agentic_hook:
             return None
 
         from litellm.rust_bridge import messages as rust_messages_bridge

litellm/llms/huggingface/embedding/handler.py

@@ -322,7 +322,7 @@ class HuggingFaceEmbedding(BaseLLM):
         task = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL)
         # print_verbose(f"{model}, {task}")
         embed_url = ""
-        if "https" in model:
+        if model.startswith(("http://", "https://")):
             embed_url = model
         elif api_base:
             embed_url = api_base

litellm/llms/huggingface/embedding/transformation.py

@@ -316,25 +316,6 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
 
         return data
 
-    def get_api_base(self, api_base: Optional[str], model: str) -> str:
-        """
-        Get the API base for the Huggingface API.
-
-        Do not add the chat/embedding/rerank extension here. Let the handler do this.
-        """
-        if "https" in model:
-            completion_url = model
-        elif api_base is not None:
-            completion_url = api_base
-        elif "HF_API_BASE" in os.environ:
-            completion_url = os.getenv("HF_API_BASE", "")
-        elif "HUGGINGFACE_API_BASE" in os.environ:
-            completion_url = os.getenv("HUGGINGFACE_API_BASE", "")
-        else:
-            completion_url = f"https://api-inference.huggingface.co/models/{model}"
-
-        return completion_url
-
     def validate_environment(
         self,
         headers: Dict,

litellm/llms/oobabooga/chat/oobabooga.py

@@ -34,7 +34,7 @@ def completion(
         optional_params=optional_params,
         litellm_params=litellm_params,
     )
-    if "https" in model:
+    if model.startswith(("http://", "https://")):
         completion_url = model
     elif api_base:
         completion_url = api_base
@@ -96,7 +96,7 @@ def embedding(
     encoding=None,
 ):
     # Create completion URL
-    if "https" in model:
+    if model.startswith(("http://", "https://")):
         embeddings_url = model
     elif api_base:
         embeddings_url = f"{api_base}/v1/embeddings"

litellm/main.py

@@ -5111,7 +5111,10 @@ def completion(  # type: ignore
     try:
         if base_url is not None:
             api_base = base_url
-        if num_retries is not None:
+        is_router_call = any("model_group" in (kwargs.get(k) or ()) for k in ("metadata", "litellm_metadata"))
+        if is_router_call:
+            max_retries = 0
+        elif num_retries is not None:
             max_retries = num_retries
         logging: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, litellm_logging_obj)
         fallbacks = fallbacks or litellm.model_fallbacks

litellm/model_prices_and_context_window_backup.json

@@ -17566,6 +17566,61 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "cache_read_input_token_cost_flex": 2e-08,
+        "cache_read_input_token_cost_priority": 5e-08,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_batches": 1.5e-07,
+        "input_cost_per_token_flex": 1.5e-07,
+        "input_cost_per_token_priority": 5.4e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_batches": 1.25e-06,
+        "output_cost_per_token_flex": 1.25e-06,
+        "output_cost_per_token_priority": 4.5e-06,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,
@@ -18233,6 +18288,60 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "vertex_ai/gemini-3.6-flash": {
+        "cache_read_input_token_cost": 1.5e-07,
+        "cache_read_input_token_cost_flex": 7.5e-08,
+        "input_cost_per_token": 1.5e-06,
+        "input_cost_per_token_batches": 7.5e-07,
+        "input_cost_per_token_flex": 7.5e-07,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 7.5e-06,
+        "output_cost_per_token": 7.5e-06,
+        "output_cost_per_token_batches": 3.75e-06,
+        "output_cost_per_token_flex": 3.75e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "input_cost_per_token_priority": 2.7e-06,
+        "output_cost_per_token_priority": 1.35e-05,
+        "cache_read_input_token_cost_priority": 2.7e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "vertex_ai/gemini-3.1-pro-preview": {
         "cache_read_input_token_cost": 2e-07,
         "cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -19585,6 +19694,63 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini/gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "cache_read_input_token_cost_flex": 2e-08,
+        "cache_read_input_token_cost_priority": 5e-08,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_batches": 1.5e-07,
+        "input_cost_per_token_flex": 1.5e-07,
+        "input_cost_per_token_priority": 5.4e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_batches": 1.25e-06,
+        "output_cost_per_token_flex": 1.25e-06,
+        "output_cost_per_token_priority": 4.5e-06,
+        "rpm": 15,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "tpm": 250000,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "gemini/gemini-3-flash-preview": {
         "cache_read_input_token_cost": 5e-08,
         "input_cost_per_audio_token": 1e-06,
@@ -19691,6 +19857,63 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini/gemini-3.6-flash": {
+        "cache_read_input_token_cost": 1.5e-07,
+        "cache_read_input_token_cost_flex": 7.5e-08,
+        "input_cost_per_token": 1.5e-06,
+        "input_cost_per_token_batches": 7.5e-07,
+        "input_cost_per_token_flex": 7.5e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 7.5e-06,
+        "output_cost_per_token": 7.5e-06,
+        "output_cost_per_token_batches": 3.75e-06,
+        "output_cost_per_token_flex": 3.75e-06,
+        "rpm": 2000,
+        "source": "https://ai.google.dev/pricing/gemini-3",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_output": false,
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "tpm": 800000,
+        "input_cost_per_token_priority": 2.7e-06,
+        "output_cost_per_token_priority": 1.35e-05,
+        "cache_read_input_token_cost_priority": 2.7e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "gemini/gemini-omni-flash-preview": {
         "input_cost_per_audio_token": 1.5e-06,
         "input_cost_per_token": 1.5e-06,
@@ -19971,6 +20194,61 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini-3.6-flash": {
+        "cache_read_input_token_cost": 1.5e-07,
+        "cache_read_input_token_cost_flex": 7.5e-08,
+        "input_cost_per_token": 1.5e-06,
+        "input_cost_per_token_batches": 7.5e-07,
+        "input_cost_per_token_flex": 7.5e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 7.5e-06,
+        "output_cost_per_token": 7.5e-06,
+        "output_cost_per_token_batches": 3.75e-06,
+        "output_cost_per_token_flex": 3.75e-06,
+        "source": "https://ai.google.dev/pricing/gemini-3",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_output": false,
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "input_cost_per_token_priority": 2.7e-06,
+        "output_cost_per_token_priority": 1.35e-05,
+        "cache_read_input_token_cost_priority": 2.7e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "gemini/gemini-2.5-pro-preview-tts": {
         "cache_read_input_token_cost": 1.25e-07,
         "cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
@@ -37232,6 +37510,61 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "vertex_ai/gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "cache_read_input_token_cost_flex": 2e-08,
+        "cache_read_input_token_cost_priority": 5e-08,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_batches": 1.5e-07,
+        "input_cost_per_token_flex": 1.5e-07,
+        "input_cost_per_token_priority": 5.4e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_batches": 1.25e-06,
+        "output_cost_per_token_flex": 1.25e-06,
+        "output_cost_per_token_priority": 4.5e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "vertex_ai/deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -597,7 +597,14 @@ async def authorize_with_server(
 ):
     _raise_if_not_oauth2(mcp_server)
     if mcp_server.authorization_url is None:
-        raise HTTPException(status_code=400, detail="MCP server authorization url is not set")
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "MCP server authorization url is not configured. Servers with no url (OpenAPI "
+                "spec or stdio) run no resource discovery, so set Authorization URL and Token URL "
+                "manually, or set Issuer to discover them from the identity provider (RFC 8414)."
+            ),
+        )
 
     if mcp_server.is_dcr_bridge:
         # Enforce S256 PKCE on both bridge arms. The relay arm forwards the validated,
@@ -702,7 +709,14 @@ async def exchange_token_with_server(
         raise HTTPException(status_code=400, detail="Unsupported grant_type")
 
     if mcp_server.token_url is None:
-        raise HTTPException(status_code=400, detail="MCP server token url is not set")
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "MCP server token url is not configured. Servers with no url (OpenAPI spec or "
+                "stdio) run no resource discovery, so set Token URL manually, or set Issuer to "
+                "discover it from the identity provider (RFC 8414)."
+            ),
+        )
 
     # The id and secret must come from the same source. When the server-side client_id wins,
     # falling back to the caller's secret pairs the persisted client with a foreign secret; the
@@ -1262,7 +1276,14 @@ async def register_client_with_server(
         return dummy_return
 
     if mcp_server.authorization_url is None:
-        raise HTTPException(status_code=400, detail="MCP server authorization url is not set")
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "MCP server authorization url is not configured. Servers with no url (OpenAPI "
+                "spec or stdio) run no resource discovery, so set Authorization URL and Token URL "
+                "manually, or set Issuer to discover them from the identity provider (RFC 8414)."
+            ),
+        )
 
     if mcp_server.registration_url is None:
         return dummy_return

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -224,6 +224,20 @@ def _uses_issuer_anchor(manual_issuer: str | None, is_discovery_auth_type: bool)
     return _blank_to_none(manual_issuer) is not None and is_discovery_auth_type
 
 
+def _has_oauth_discovery_source(server_url: str | None, use_issuer_anchor: bool) -> bool:
+    """Whether the server has any source OAuth discovery can fetch metadata from.
+
+    Resource-rooted discovery (RFC 9728) is fetched from the server ``url``, so spec-only
+    (OpenAPI) and stdio servers, which have none, could never discover: their OAuth endpoints
+    stayed unset unless entered manually and ``/authorize`` served its 400 with no hint of why.
+    An admin-pinned issuer is a trust anchor in its own right (RFC 8414 section 3.3) whose
+    metadata fetch does not touch the resource at all, so an anchored server can discover with
+    no ``url``. Called by both build paths (config and DB) so the two cannot disagree on when
+    discovery is reachable.
+    """
+    return bool(server_url) or use_issuer_anchor
+
+
 def _endpoints_yield_to_issuer(
     issuer: str | None,
     is_discovery_auth_type: bool,
@@ -610,6 +624,34 @@ def _passthrough_token_from_mcp_auth_header(
     return None
 
 
+async def _materialize_auth_headers(auth: httpx.Auth | None) -> dict[str, str] | None:
+    """Extract the header a resolved ``httpx.Auth`` would set, as a plain dict, or None.
+
+    OpenAPI tool closures egress through ``AsyncHTTPHandler`` methods that accept headers but no
+    ``auth``, so a resolved credential must be materialized into a header value. Driving one step
+    of the auth's own flow (against a throwaway request that is never sent) keeps this generic
+    across every auth shape without per-class branching; ``header_name`` is the resolver-arm
+    convention for "this auth sets a header" (``NoOpAuth`` has none and yields nothing to apply).
+    The materialized value is point-in-time: flow behaviors past the first request, like the M2M
+    one-shot 401 refetch, do not apply on this arm.
+    """
+    if auth is None:
+        return None
+    header_name = getattr(auth, "header_name", None)
+    if not isinstance(header_name, str) or not header_name:
+        return None
+    probe = httpx.Request("GET", "http://localhost/")
+    flow = auth.async_auth_flow(probe)
+    try:
+        first_request = await flow.__anext__()
+    except StopAsyncIteration:
+        return None
+    finally:
+        await flow.aclose()
+    header_value = first_request.headers.get(header_name)
+    return {header_name: header_value} if header_value else None
+
+
 def _consumes_caller_authorization(server: MCPServer) -> bool:
     """True when this server's egress forwards the caller's request-wide ``Authorization`` upstream:
     the client-forwarded token modes, legacy OAuth pass-through, and legacy upstream-delegated
@@ -1226,7 +1268,12 @@ class MCPServerManager:
             manual_token_url = _blank_to_none(server_config.get("token_url"))
             manual_registration_url = _blank_to_none(server_config.get("registration_url"))
             is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
-            use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type)
+            obo_needs_discovery = self._obo_needs_endpoint_discovery(
+                auth_type,
+                server_config.get("token_exchange_endpoint"),
+                manual_token_url,
+            )
+            use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type or obo_needs_discovery)
             manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer(
                 manual_issuer,
                 is_discovery_auth_type,
@@ -1234,17 +1281,12 @@ class MCPServerManager:
                 manual_token_url,
                 manual_registration_url,
             )
-            should_discover = bool(server_url) and (
-                is_discovery_auth_type
-                or self._obo_needs_endpoint_discovery(
-                    auth_type,
-                    server_config.get("token_exchange_endpoint"),
-                    manual_token_url,
-                )
+            should_discover = _has_oauth_discovery_source(server_url, use_issuer_anchor) and (
+                is_discovery_auth_type or obo_needs_discovery
             )
             if not should_discover:
                 mcp_oauth_metadata = None
-            elif manual_issuer is not None and is_discovery_auth_type:
+            elif use_issuer_anchor and manual_issuer is not None:
                 mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url)
             else:
                 mcp_oauth_metadata = await self._descovery_metadata(
@@ -1640,7 +1682,7 @@ class MCPServerManager:
         token_exchange_endpoint: Optional[str],
     ) -> Optional[MCPOAuthMetadata]:
         has_all_upstream_oauth_fields = bool(manual_authorization_url and manual_token_url and scopes)
-        needs_discovery = bool(server_url) and (
+        needs_discovery = _has_oauth_discovery_source(server_url, use_issuer_anchor) and (
             (is_discovery_auth_type and not has_all_upstream_oauth_fields)
             or self._obo_needs_endpoint_discovery(auth_type, token_exchange_endpoint, manual_token_url)
         )
@@ -1759,13 +1801,17 @@ class MCPServerManager:
         manual_token_url = _blank_to_none(mcp_server.token_url)
         manual_registration_url = _blank_to_none(mcp_server.registration_url)
         is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
-        use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type)
-        manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer(
-            manual_issuer, is_discovery_auth_type, manual_authorization_url, manual_token_url, manual_registration_url
-        )
         token_exchange_endpoint = mcp_server.token_exchange_endpoint or (
             credentials_dict.get("token_exchange_endpoint") if credentials_dict else None
         )
+        use_issuer_anchor = _uses_issuer_anchor(
+            manual_issuer,
+            is_discovery_auth_type
+            or self._obo_needs_endpoint_discovery(auth_type, token_exchange_endpoint, manual_token_url),
+        )
+        manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer(
+            manual_issuer, is_discovery_auth_type, manual_authorization_url, manual_token_url, manual_registration_url
+        )
         gated_oauth_metadata = await self._resolve_table_oauth_metadata(
             mcp_server=mcp_server,
             auth_type=auth_type,
@@ -1943,7 +1989,7 @@ class MCPServerManager:
         family: discovered ``authorization_url``/``token_url``/``scopes`` otherwise live only on
         the in-memory registry entry, which is rebuilt on every client connect (the DCR reuse path
         calls ``update_server``) and on every post-write DB reload, so one failed re-discovery
-        serves 400 "authorization url is not set" from /authorize until a later rebuild succeeds.
+        serves the 400 "authorization url is not configured" from /authorize until a later rebuild succeeds.
         Only fills row fields that are currently empty, never persists origin-fallback guesses
         (RFC 9728/8414-advertised metadata only), and deliberately skips ``registration_url``
         because ``_dcr_bridge_relays_client_registration`` keys off that column. Best-effort: a
@@ -4705,6 +4751,61 @@ class MCPServerManager:
             )
         return oauth2_headers
 
+    async def resolve_openapi_upstream_auth(
+        self,
+        *,
+        mcp_server: MCPServer,
+        oauth2_headers: dict[str, str] | None,
+        raw_headers: dict[str, str] | None,
+        mcp_auth_header: str | dict[str, str] | None,
+        user_api_key_auth: UserAPIKeyAuth | None,
+        forwarded_headers: dict[str, str] | None,
+    ) -> tuple[dict[str, str] | None, dict[str, str] | None]:
+        """Resolve the gateway-owned upstream credential for a spec_path (OpenAPI) tool call.
+
+        OpenAPI tools egress through a plain httpx call assembled from ContextVars, never through
+        ``_create_mcp_client``, so the v2 resolver graft there does not run for them and a resolved
+        credential (authorization_code's stored per-user token, client_credentials' minted M2M
+        token, token_exchange's exchanged token, passthrough's forwarded caller token) must be
+        materialized into headers here. Returns ``(resolved_auth_headers, forwarded_headers)``:
+        the resolved headers are authoritative over every other Authorization source (the same
+        rule ``_resolve_v2_auth`` applies on the MCPClient path) and ``forwarded_headers`` comes
+        back with any header the resolver claimed already dropped. Unmigrated (v1) servers resolve
+        through the stored-token lookup instead, and a missing per-user credential raises the same
+        discovery challenge the MCPClient path serves, rather than egressing unauthenticated.
+
+        The resolved headers carry only credentials the gateway itself resolved (a stored per-user
+        token, a minted or exchanged token). Caller-supplied ``oauth2_headers`` are never promoted
+        into them: on the v2 arm they feed only subject-token extraction (the designed RFC 8693
+        input), and on the v1 arm their presence disables the stored lookup entirely, so a
+        caller's gateway credential can never displace a per-server BYOK header or leak upstream
+        as the resolved credential.
+        """
+        spec = to_server_spec(mcp_server)
+        if spec is None:
+            if oauth2_headers:
+                return None, forwarded_headers
+            stored_headers = await self._resolve_oauth2_headers_for_tool_call(mcp_server, None, user_api_key_auth)
+            return stored_headers, forwarded_headers
+
+        subject_token: str | None = None
+        if isinstance(spec.config, (TokenExchangeConfig, IdJagConfig)):
+            subject_token = self._extract_bearer_token(oauth2_headers, raw_headers)
+        elif isinstance(spec.config, PassthroughConfig):
+            inbound_token, forwarded_headers = _take_forwarded_authorization(forwarded_headers)
+            per_server_token = _passthrough_token_from_mcp_auth_header(mcp_auth_header)
+            subject_token = per_server_token if per_server_token is not None else inbound_token
+
+        resolved_auth, forwarded_headers = await self._resolve_v2_auth(
+            server=mcp_server,
+            spec=spec,
+            provider=self._cred_provider,
+            subject_token=subject_token,
+            user_api_key_auth=user_api_key_auth,
+            extra_headers=forwarded_headers,
+        )
+        return await _materialize_auth_headers(resolved_auth), forwarded_headers
+
     async def _gather_openapi_tool_tasks(
         self,
         tasks: list[Any],
@@ -4796,6 +4897,7 @@ class MCPServerManager:
             )
             tasks.append(during_hook_task)
 
+        caller_oauth2_headers = oauth2_headers
         oauth2_headers = await self._resolve_oauth2_headers_for_tool_call(mcp_server, oauth2_headers, user_api_key_auth)
 
         # For OpenAPI servers, call the tool handler directly instead of via MCP client
@@ -4813,22 +4915,32 @@ class MCPServerManager:
             auth_header_value = (
                 _format_byok_openapi_auth_header(mcp_server, mcp_auth_header) if mcp_auth_header else None
             )
-            forwarded_headers = _openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth)
+            resolved_auth_headers, forwarded_headers = await self.resolve_openapi_upstream_auth(
+                mcp_server=mcp_server,
+                oauth2_headers=caller_oauth2_headers,
+                raw_headers=raw_headers,
+                mcp_auth_header=mcp_auth_header,
+                user_api_key_auth=user_api_key_auth,
+                forwarded_headers=_openapi_forwarded_extra_headers(mcp_server, raw_headers, user_api_key_auth),
+            )
 
             async def _call_openapi_via_handler():
                 from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
                     _request_auth_header,
                     _request_extra_headers,
+                    _request_resolved_auth_headers,
                 )
 
                 auth_token = _request_auth_header.set(auth_header_value)
                 extra_token = _request_extra_headers.set(forwarded_headers)
+                resolved_token = _request_resolved_auth_headers.set(resolved_auth_headers)
                 try:
                     async with self._limit_outbound_concurrency(mcp_server):
                         return await self._call_openapi_tool_handler(mcp_server, name, arguments)
                 finally:
                     _request_auth_header.reset(auth_token)
                     _request_extra_headers.reset(extra_token)
+                    _request_resolved_auth_headers.reset(resolved_token)
 
             tasks.append(asyncio.create_task(_call_openapi_via_handler()))
         else:

litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py

@@ -62,6 +62,14 @@ _request_extra_headers: contextvars.ContextVar[Optional[Dict[str, str]]] = conte
     "_request_extra_headers", default=None
 )
 
+# Per-request headers carrying the gateway-resolved upstream credential
+# (stored per-user OAuth token, minted M2M token, exchanged OBO token).
+# Set from MCPServerManager.resolve_openapi_upstream_auth; authoritative
+# over every other Authorization source in _merge_openapi_tool_request_headers.
+_request_resolved_auth_headers: contextvars.ContextVar[dict[str, str] | None] = contextvars.ContextVar(
+    "_request_resolved_auth_headers", default=None
+)
+
 
 def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
     """Ensure path params cannot introduce directory traversal."""
@@ -294,10 +302,15 @@ def _merge_openapi_tool_request_headers(
     """Merge static closure headers with per-request ContextVar overrides.
 
     Precedence (highest to lowest):
-        1. ``_request_auth_header`` — BYOK override of ``Authorization``
-        2. ``static_headers`` — operator-configured headers baked into the
+        1. ``_request_resolved_auth_headers`` — the gateway-resolved upstream
+           credential (stored per-user OAuth token, minted M2M token,
+           exchanged OBO token). The resolver is authoritative: a BYOK or
+           forwarded ``Authorization`` must not shadow it, mirroring
+           ``_resolve_v2_auth`` on the MCPClient path
+        2. ``_request_auth_header`` — BYOK override of ``Authorization``
+        3. ``static_headers`` — operator-configured headers baked into the
            tool closure at registration time
-        3. ``_request_extra_headers`` — per-request headers forwarded from
+        4. ``_request_extra_headers`` — per-request headers forwarded from
            the MCP caller (allowlisted by ``MCPServer.extra_headers``)
 
     This matches the existing MCP invariant in
@@ -323,6 +336,12 @@ def _merge_openapi_tool_request_headers(
             del effective_headers[existing]
         effective_headers["Authorization"] = override_auth
 
+    resolved_auth_headers = _request_resolved_auth_headers.get() or {}
+    for name, value in resolved_auth_headers.items():
+        for existing in [k for k in effective_headers if k.lower() == name.lower()]:
+            del effective_headers[existing]
+        effective_headers[name] = value
+
     return effective_headers
 
 

litellm/proxy/_experimental/mcp_server/outbound_credentials/sso_assertion_store.py

@@ -0,0 +1,214 @@
+"""Store for the enterprise IdP identity assertion captured at SSO login (EMA).
+
+The ``oauth2_id_jag`` egress arm needs the user's IdP ``id_token`` as its RFC 8693
+``subject_token``. A front-door client holds an identity-only ``llm_session_`` bearer, not an
+IdP assertion, so the assertion captured at the one SSO login is the only usable subject
+source for it. This module owns both sides of that state: the SSO callback persists here
+(write-through to the DB so a login on one pod is visible to every pod) and the resolver
+seam reads back by ``user_id``. Retention is gated on an ``oauth2_id_jag`` server actually
+being registered, so a gateway with no EMA upstream never stores bearer material.
+
+The row is one encrypted payload per user, latest login wins. ``expires_at`` mirrors the
+id_token ``exp`` claim and is judged by the reader, never enforced by deletion here: an
+expired assertion with a refresh token is still renewable, and the DB row is the source of
+truth, the same contract as the per-user OAuth credential store.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+import jwt
+from pydantic import BaseModel, ConfigDict, SecretStr, TypeAdapter, ValidationError
+
+from litellm._logging import verbose_proxy_logger
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+_ASSERTION_DECRYPT_LOG_KEY = "sso_identity_assertion"
+_STR_ADAPTER: TypeAdapter[str] = TypeAdapter(str)
+_MAYBE_STR_ADAPTER: TypeAdapter[str | None] = TypeAdapter(str | None)
+
+
+class SSOIdentityAssertion(BaseModel):
+    """The IdP material an EMA exchange needs: ``id_token`` is the RFC 8693 subject token,
+    ``expires_at`` bounds its usefulness, and the refresh token renews it without re-login."""
+
+    model_config = ConfigDict(frozen=True)
+
+    id_token: SecretStr
+    refresh_token: SecretStr | None = None
+    issuer: str | None = None
+    expires_at: datetime | None = None
+
+
+class _IdTokenClaims(BaseModel):
+    exp: float | None = None
+    iss: str | None = None
+
+
+class _StoredAssertionPayload(BaseModel):
+    id_token: str
+    refresh_token: str | None = None
+    issuer: str | None = None
+    expires_at: datetime | None = None
+
+
+def assertion_from_sso_login(id_token: object, refresh_token: object) -> SSOIdentityAssertion | None:
+    """The typed carrier built where the raw token response exists; ``None`` when the provider
+    sent no id_token or sent one that is not a decodable JWT, since neither is exchangeable
+    under EMA. Inputs are ``object`` because they come straight from the provider's untyped
+    token response; this is the one boundary that validates them. The token arrived over TLS
+    from the IdP's own token endpoint, so claims are read without signature verification,
+    matching how the SSO callback already decodes it for identity."""
+    raw_id_token = id_token if isinstance(id_token, str) and id_token else None
+    if raw_id_token is None:
+        return None
+    raw_refresh_token = refresh_token if isinstance(refresh_token, str) and refresh_token else None
+    try:
+        claims = _IdTokenClaims.model_validate(jwt.decode(raw_id_token, options={"verify_signature": False}))
+        expires_at = datetime.fromtimestamp(claims.exp, tz=timezone.utc) if claims.exp is not None else None
+    except Exception:  # noqa: BLE001  # decode failure = not retainable; never raise into login
+        verbose_proxy_logger.warning(
+            "SSO id_token could not be decoded or its claims were unusable; not retaining it for EMA egress."
+        )
+        return None
+    return SSOIdentityAssertion(
+        id_token=SecretStr(raw_id_token),
+        refresh_token=SecretStr(raw_refresh_token) if raw_refresh_token else None,
+        issuer=claims.iss,
+        expires_at=expires_at,
+    )
+
+
+async def ema_assertion_retention_enabled() -> bool:
+    """Whether any MCP server uses ``oauth2_id_jag``, evaluated per login so the gateway only
+    retains bearer material while an EMA upstream exists to spend it on. Judged against the two
+    configuration authorities: the pod-local config declaration and the shared DB row. The
+    in-memory registry is deliberately not consulted in either direction; it is a per-process
+    snapshot of the DB state that can be stale both ways (a server added on another pod would
+    silently drop the write, one removed on another pod would keep retaining bearer material),
+    and a gate guarding a shared-DB write must judge against that storage's authority."""
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415  # avoids import cycle
+        global_mcp_server_manager,
+    )
+    from litellm.proxy.proxy_server import prisma_client  # noqa: PLC0415  # runtime global
+    from litellm.types.mcp import MCPAuth  # noqa: PLC0415  # runtime global
+
+    config_servers = global_mcp_server_manager.config_mcp_servers.values()
+    if any(server.auth_type == MCPAuth.oauth2_id_jag for server in config_servers):
+        return True
+    if prisma_client is None:
+        return False
+    row = await prisma_client.db.litellm_mcpservertable.find_first(where={"auth_type": MCPAuth.oauth2_id_jag.value})
+    return row is not None
+
+
+async def persist_sso_identity_assertion(user_id: str, assertion: SSOIdentityAssertion) -> None:
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper  # noqa: PLC0415  # runtime global
+    from litellm.proxy.proxy_server import prisma_client  # noqa: PLC0415  # runtime global
+
+    if prisma_client is None:
+        return
+    payload: dict[str, str] = {
+        "id_token": assertion.id_token.get_secret_value(),
+        **({"refresh_token": assertion.refresh_token.get_secret_value()} if assertion.refresh_token else {}),
+        **({"issuer": assertion.issuer} if assertion.issuer else {}),
+        **({"expires_at": assertion.expires_at.isoformat()} if assertion.expires_at else {}),
+    }
+    encoded = _STR_ADAPTER.validate_python(encrypt_value_helper(json.dumps(payload)))
+    await prisma_client.db.litellm_ssoidentityassertion.upsert(
+        where={"user_id": user_id},
+        data={
+            "create": {"user_id": user_id, "assertion_b64": encoded},
+            "update": {"assertion_b64": encoded},
+        },
+    )
+
+
+async def fetch_sso_identity_assertion(user_id: str) -> SSOIdentityAssertion | None:
+    """The stored assertion for ``user_id``, or ``None`` when absent, undecryptable (salt-key
+    rotation), or unparseable. Expiry is not judged here; the reader owns that policy."""
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper  # noqa: PLC0415  # runtime global
+    from litellm.proxy.proxy_server import prisma_client  # noqa: PLC0415  # runtime global
+
+    if prisma_client is None:
+        return None
+    row = await prisma_client.db.litellm_ssoidentityassertion.find_unique(where={"user_id": user_id})
+    if row is None:
+        return None
+    raw = _MAYBE_STR_ADAPTER.validate_python(
+        decrypt_value_helper(row.assertion_b64, _ASSERTION_DECRYPT_LOG_KEY, exception_type="debug")
+    )
+    if raw is None:
+        return None
+    try:
+        payload = _StoredAssertionPayload.model_validate_json(raw)
+    except ValidationError:
+        verbose_proxy_logger.warning(
+            "Stored SSO identity assertion for user_id=%s could not be parsed; treating as absent.", user_id
+        )
+        return None
+    return SSOIdentityAssertion(
+        id_token=SecretStr(payload.id_token),
+        refresh_token=SecretStr(payload.refresh_token) if payload.refresh_token else None,
+        issuer=payload.issuer,
+        expires_at=payload.expires_at,
+    )
+
+
+async def rotate_sso_identity_assertions_master_key(prisma_client: PrismaClient, new_master_key: str) -> None:
+    """Re-encrypt every stored assertion under ``new_master_key`` during a salt-key rotation,
+    mirroring the sibling per-user credential tables; an unreadable row is skipped so one
+    corrupt row does not abort the rotation. Rows are decrypted one at a time inside the loop
+    so the whole table's plaintext is never held in memory at once."""
+    from prisma.models import LiteLLM_SSOIdentityAssertion as AssertionRow  # noqa: PLC0415  # generated at runtime
+
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import (  # noqa: PLC0415  # runtime global
+        decrypt_value_helper,
+        encrypt_value_helper,
+    )
+
+    async def _rotate_row(row: AssertionRow) -> bool:
+        plaintext = _MAYBE_STR_ADAPTER.validate_python(
+            decrypt_value_helper(row.assertion_b64, _ASSERTION_DECRYPT_LOG_KEY, exception_type="debug")
+        )
+        if plaintext is None:
+            verbose_proxy_logger.warning(
+                "rotate_sso_identity_assertions_master_key: could not decrypt assertion for user_id=%s, skipping",
+                row.user_id,
+            )
+            return False
+        re_encrypted = _STR_ADAPTER.validate_python(encrypt_value_helper(plaintext, new_encryption_key=new_master_key))
+        await prisma_client.db.litellm_ssoidentityassertion.update(
+            where={"user_id": row.user_id},
+            data={"assertion_b64": re_encrypted},
+        )
+        return True
+
+    rows = await prisma_client.db.litellm_ssoidentityassertion.find_many()
+    outcomes = [await _rotate_row(row) for row in rows]
+    verbose_proxy_logger.info(
+        "rotate_sso_identity_assertions_master_key: rotated %d row(s), skipped %d",
+        sum(outcomes),
+        len(outcomes) - sum(outcomes),
+    )
+
+
+async def retain_sso_identity_assertion_for_ema(user_id: str, assertion: SSOIdentityAssertion | None) -> None:
+    """The SSO-callback hook: a no-op unless there is material AND an EMA server is registered.
+    A store failure is logged and swallowed because the login itself must not fail on an
+    egress-side write; the cost of a miss is a 401 challenge at the EMA upstream, not a lockout."""
+    if assertion is None:
+        return
+    try:
+        if not await ema_assertion_retention_enabled():
+            return
+        await persist_sso_identity_assertion(user_id, assertion)
+    except Exception as exc:  # noqa: BLE001  # the login itself must not fail on an egress-side write
+        verbose_proxy_logger.warning(
+            "Failed to persist the SSO identity assertion for EMA egress (user_id=%s): %s", user_id, exc
+        )

litellm/proxy/_experimental/mcp_server/server.py

@@ -376,6 +376,7 @@ if MCP_AVAILABLE:
     from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
         _request_auth_header,
         _request_extra_headers,
+        _request_resolved_auth_headers,
     )
     from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport
     from litellm.proxy._experimental.mcp_server.tool_registry import (
@@ -2785,13 +2786,29 @@ if MCP_AVAILABLE:
                             forwarded_headers = {}
                         forwarded_headers[header_name] = value
 
+            resolved_auth_headers: dict[str, str] | None = None
+            if mcp_server:
+                (
+                    resolved_auth_headers,
+                    forwarded_headers,
+                ) = await global_mcp_server_manager.resolve_openapi_upstream_auth(
+                    mcp_server=mcp_server,
+                    oauth2_headers=oauth2_headers,
+                    raw_headers=raw_headers,
+                    mcp_auth_header=mcp_auth_header,
+                    user_api_key_auth=user_api_key_auth,
+                    forwarded_headers=forwarded_headers,
+                )
+
             _auth_token = _request_auth_header.set(auth_header_value)
             _extra_token = _request_extra_headers.set(forwarded_headers)
+            _resolved_token = _request_resolved_auth_headers.set(resolved_auth_headers)
             try:
                 local_content = await _handle_local_mcp_tool(name, arguments)
             finally:
                 _request_auth_header.reset(_auth_token)
                 _request_extra_headers.reset(_extra_token)
+                _request_resolved_auth_headers.reset(_resolved_token)
             response = CallToolResult(content=cast(Any, local_content), isError=False)
 
         # Try managed MCP server tool (pass the full prefixed name)

litellm/proxy/_types.py

@@ -2298,6 +2298,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="max response size in MB, if a response is larger than this size it will be rejected",
     )
+    proxy_config_reload_interval_seconds: int = Field(
+        30,
+        gt=0,
+        description="how often (in seconds) each pod reloads config-in-DB objects (models, credentials, guardrails, etc.) when store_model_in_db is enabled; lower values speed up multi-pod convergence at the cost of more DB load. Applied on proxy startup",
+    )
     cancel_on_disconnect: Optional[bool] = Field(
         None,
         description="cancel the in-flight upstream LLM request (non-streaming) when the client disconnects, freeing backend capacity (e.g. a vLLM GPU slot); the request is logged as a 499 failure",
@@ -4048,6 +4053,7 @@ class JWTAuthBuilderResult(TypedDict):
     token: str
     team_id: Optional[str]
     user_id: Optional[str]
+    user_email: str | None
     end_user_id: Optional[str]
     org_id: Optional[str]
     team_membership: Optional[LiteLLM_TeamMembership]

litellm/proxy/a2a/agent_card.py

@@ -7,23 +7,46 @@ the base; specific fields are replaced so all traffic flows through the proxy
 and uses LiteLLM auth.
 """
 
+import re
 from copy import deepcopy
-from typing import Any, Dict, List, Mapping
+from typing import Any, Dict, List, Literal, Mapping
+
+SupportedA2AVersion = Literal["0.3", "1.0"]
 
 # Protocol versions LiteLLM can serve to A2A clients. The admin pins one per agent;
 # responses are normalized to it regardless of the upstream agent's own version.
-SUPPORTED_A2A_PROTOCOL_VERSIONS = ("0.3", "1.0")
+SUPPORTED_A2A_PROTOCOL_VERSIONS: tuple[SupportedA2AVersion, ...] = ("0.3", "1.0")
 
 # Default served version when the agent card does not pin one.
 LITELLM_A2A_PROTOCOL_VERSION = "1.0"
 
 
+_PROTOCOL_VERSION_PATTERN = re.compile(
+    r"^(\d+\.\d+)(?:\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)?$"
+)
+
+
+def normalize_protocol_version(version: object) -> SupportedA2AVersion | None:
+    """Map a raw ``protocolVersion`` value to the supported canonical major.minor version.
+
+    Accepts the bare major.minor convention of the 1.0 spec (``"0.3"``, ``"1.0"``) and the
+    full semver forms older SDKs emit (``"0.3.0"``, ``"1.0.1"``, including prerelease and
+    build suffixes like ``"0.3.0-rc1"``). Malformed strings, versions outside the
+    supported set, and non-strings yield ``None``.
+    """
+    if not isinstance(version, str):
+        return None
+    match = _PROTOCOL_VERSION_PATTERN.match(version)
+    if match is None:
+        return None
+    major_minor = match.group(1)
+    return next((supported for supported in SUPPORTED_A2A_PROTOCOL_VERSIONS if supported == major_minor), None)
+
+
 def resolve_served_protocol_version(card: Mapping[str, Any] | None) -> str:
     """Return the validated protocol version an agent card pins, else the default."""
-    version = card.get("protocolVersion") if card else None
-    if version in SUPPORTED_A2A_PROTOCOL_VERSIONS:
-        return version
-    return LITELLM_A2A_PROTOCOL_VERSION
+    normalized = normalize_protocol_version(card.get("protocolVersion") if card else None)
+    return normalized if normalized is not None else LITELLM_A2A_PROTOCOL_VERSION
 
 
 # Security scheme exposed by the LiteLLM-fronted agent card. Always replaces

litellm/proxy/a2a/version_convert.py

@@ -30,6 +30,7 @@ from typing import Callable, Literal, Union
 from pydantic import BaseModel
 
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy.a2a.agent_card import normalize_protocol_version
 
 A2AVersion = Literal["0.3", "1.0"]
 RequestId = Union[str, int, None]
@@ -103,16 +104,14 @@ def normalize_request_params(params: JsonDict, served: A2AVersion, *, method: st
 def _detect_card_version(card: JsonDict) -> A2AVersion:
     """Infer the wire version of an agent card dict.
 
-    ``protocolVersion`` is the authoritative indicator; fall back to presence of
-    ``supportedInterfaces`` (a 1.0-only field) only when the explicit field is absent.
-    Cards that set ``protocolVersion: "0.3"`` or carry neither signal are treated as 0.3.
+    ``protocolVersion`` is the authoritative indicator; semver values normalize to
+    their major.minor (``"0.3.0"`` -> ``"0.3"``). Fall back to presence of
+    ``supportedInterfaces`` (a 1.0-only field) only when the explicit field is
+    absent or unrecognized; cards carrying neither signal are treated as 0.3.
     """
-    pv = card.get("protocolVersion")
-    if pv == "1.0":
-        return "1.0"
-    if pv == "0.3":
-        return "0.3"
-    # No protocolVersion field: use structural heuristic.
+    normalized = normalize_protocol_version(card.get("protocolVersion"))
+    if normalized is not None:
+        return normalized
     return "1.0" if "supportedInterfaces" in card else "0.3"
 
 

litellm/proxy/agent_endpoints/endpoints.py

@@ -23,6 +23,7 @@ from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKey
 from litellm.proxy.a2a.agent_card import (
     SUPPORTED_A2A_PROTOCOL_VERSIONS,
     merge_agent_card,
+    normalize_protocol_version,
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
@@ -51,7 +52,7 @@ def _proxy_base_url(http_request: Request) -> str:
 def _validate_protocol_version(upstream_card: Mapping[str, Any] | None) -> None:
     """Reject an agent card pinning an unsupported A2A protocol version."""
     version = upstream_card.get("protocolVersion") if upstream_card else None
-    if version is not None and version not in SUPPORTED_A2A_PROTOCOL_VERSIONS:
+    if version is not None and normalize_protocol_version(version) is None:
         raise HTTPException(
             status_code=400,
             detail=(

litellm/proxy/auth/auth_utils.py

@@ -3,7 +3,7 @@ import re
 import sys
 from functools import lru_cache
 from logging import Logger
-from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Tuple, Union
+from typing import Any, Dict, FrozenSet, Iterator, List, Mapping, Optional, Tuple, Union
 
 from fastapi import HTTPException, Request, status
 
@@ -12,7 +12,12 @@ from litellm import Router, provider_list
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
+from litellm.litellm_core_utils.url_utils import (
+    SSRFError,
+    is_url_destination_allowed_by_host,
+    provider_url_destination_candidates,
+    validate_url,
+)
 from litellm.proxy._types import *
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
@@ -290,6 +295,7 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = (
     "use_ssl",
     # SDK-only field; also rejected outright in is_request_body_safe.
     "model_list",
+    "vertex_ai_credentials",
     # Observability credentials, hosts, and project identifiers: derived
     # from the canonical ``_supported_callback_params`` allowlist so new
     # integrations are covered automatically. Sorted for stable iteration
@@ -342,6 +348,60 @@ def _check_banned_params(
         )
 
 
+_FALLBACK_FIELDS: tuple[str, ...] = (
+    "fallbacks",
+    "context_window_fallbacks",
+    "content_policy_fallbacks",
+)
+
+
+def _iter_fallback_field_values(request_body: Mapping[str, object]) -> Iterator[object]:
+    override = request_body.get("router_settings_override")
+    for source in (request_body, override):
+        if isinstance(source, Mapping):
+            for field in _FALLBACK_FIELDS:
+                yield source.get(field)
+
+
+def _iter_fallback_targets(value: object, depth: int) -> Iterator[str | Mapping[str, object]]:
+    if depth > 2 * litellm.ROUTER_MAX_FALLBACKS:
+        raise ValueError("Rejected Request: fallback nesting exceeds the allowed validation depth.")
+    if not isinstance(value, list):
+        return
+    for item in value:
+        if isinstance(item, str):
+            yield item
+        elif isinstance(item, Mapping):
+            values = tuple(item.values())
+            if not (values and all(isinstance(v, list) for v in values)):
+                yield item
+            if isinstance(item.get("model"), str):
+                for field in _FALLBACK_FIELDS:
+                    yield from _iter_fallback_targets(item.get(field), depth + 1)
+            else:
+                for target_list in values:
+                    yield from _iter_fallback_targets(target_list, depth + 1)
+
+
+def iter_request_fallback_targets(request_body: Mapping[str, object]) -> Iterator[str | Mapping[str, object]]:
+    for value in _iter_fallback_field_values(request_body):
+        yield from _iter_fallback_targets(value, 0)
+
+
+def _reject_url_valued_fallback_target(value: str) -> None:
+    allowed_hosts = getattr(litellm, "provider_url_destination_allowed_hosts", []) or []
+    for candidate in provider_url_destination_candidates(value):
+        if not candidate.lower().startswith(("http://", "https://")):
+            continue
+        if is_url_destination_allowed_by_host(candidate, allowed_hosts):
+            continue
+        raise ValueError(
+            f"Rejected Request: URL-valued fallback destination '{value}' is not allowed. "
+            "Configure custom endpoints with api_base instead, or add the destination host to "
+            "`provider_url_destination_allowed_hosts` in litellm_settings."
+        )
+
+
 def is_request_body_safe(request_body: dict, general_settings: dict, llm_router: Optional[Router], model: str) -> bool:
     """
     Check if the request body is safe.
@@ -379,6 +439,14 @@ def is_request_body_safe(request_body: dict, general_settings: dict, llm_router:
         metadata = _coerce_metadata_to_dict(request_body.get(metadata_key))
         if metadata is not None:
             _check_banned_params(metadata, general_settings, llm_router, model)
+    for target in iter_request_fallback_targets(request_body):
+        if isinstance(target, dict):
+            _check_banned_params(target, general_settings, llm_router, model)
+            target_model = target.get("model")
+            if isinstance(target_model, str):
+                _reject_url_valued_fallback_target(target_model)
+        elif isinstance(target, str):
+            _reject_url_valued_fallback_target(target)
     litellm_params = _coerce_metadata_to_dict(request_body.get("litellm_params"))
     if litellm_params is not None:
         litellm_params_metadata = _coerce_metadata_to_dict(litellm_params.get("metadata"))

litellm/proxy/auth/handle_jwt.py

@@ -1155,6 +1155,7 @@ class JWTAuthManager:
         org_id: Optional[str],
         api_key: str,
         jwt_valid_token: Optional[dict] = None,
+        user_email: str | None = None,
     ) -> Optional[JWTAuthBuilderResult]:
         """Check admin status and route access permissions"""
         if not jwt_handler.is_admin(scopes=scopes):
@@ -1179,6 +1180,7 @@ class JWTAuthManager:
             token=api_key,
             team_id=None,
             user_id=user_id,
+            user_email=user_email,
             end_user_id=None,
             org_id=org_id,
             team_membership=None,
@@ -2068,7 +2070,7 @@ class JWTAuthManager:
 
         # Check admin access
         admin_result = await JWTAuthManager.check_admin_access(
-            jwt_handler, scopes, route, user_id, org_id, api_key, jwt_valid_token
+            jwt_handler, scopes, route, user_id, org_id, api_key, jwt_valid_token, user_email=user_email
         )
         if admin_result:
             await JWTAuthManager._attach_team_from_header_for_admin(
@@ -2303,6 +2305,7 @@ class JWTAuthManager:
             team_id=team_id,
             team_object=team_object,
             user_id=user_id,
+            user_email=(user_object.user_email if user_object is not None and user_object.user_email else user_email),
             user_object=user_object,
             org_id=resolved_org_id,  # Use resolved org_id (from alias lookup if applicable)
             org_object=org_object,

litellm/proxy/auth/user_api_key_auth.py

@@ -14,7 +14,7 @@ import secrets
 
 import orjson
 from datetime import datetime, timezone
-from typing import Any, Dict, Iterator, NamedTuple, List, Optional, Protocol, Tuple, Union, cast
+from typing import Any, Dict, NamedTuple, List, Optional, Protocol, Tuple, Union, cast
 
 import fastapi
 from fastapi import HTTPException, Request, WebSocket, status
@@ -58,6 +58,7 @@ from litellm.proxy.auth.auth_utils import (
     get_model_from_request,
     get_request_route,
     get_request_route_template,
+    iter_request_fallback_targets,
     normalize_request_route,
     pre_db_read_auth_checks,
     route_in_additonal_public_routes,
@@ -1011,7 +1012,7 @@ def _ensure_parent_otel_span_on_request_state(request: Request) -> None:
         return
     if getattr(request.state, "parent_otel_span", None) is not None:
         return
-    start_time = datetime.now()
+    start_time = datetime.now(timezone.utc)
     try:
         request.state.litellm_received_at = start_time
     except Exception:
@@ -1061,7 +1062,7 @@ async def _user_api_key_auth_builder(
     # Prefer the receive-instant stamped by the early helper in
     # user_api_key_auth (before body parse) — overwriting it would shorten
     # the preprocessing-duration measurement by the body-parse window.
-    start_time = getattr(request.state, "litellm_received_at", None) or datetime.now()
+    start_time = getattr(request.state, "litellm_received_at", None) or datetime.now(timezone.utc)
     try:
         request.state.litellm_received_at = start_time
     except Exception:
@@ -1255,6 +1256,7 @@ async def _user_api_key_auth_builder(
                     team_id = result["team_id"]
                     team_object = result["team_object"]
                     user_id = result["user_id"]
+                    user_email = result["user_email"]
                     user_object = result["user_object"]
                     end_user_id = result["end_user_id"]
                     org_id = result["org_id"]
@@ -1279,6 +1281,7 @@ async def _user_api_key_auth_builder(
                             api_key=None,
                             user_role=LitellmUserRoles.PROXY_ADMIN,
                             user_id=user_id,
+                            user_email=user_email,
                             team_id=team_id,
                             team_alias=(team_object.team_alias if team_object is not None else None),
                             team_tpm_limit=(team_object.tpm_limit if team_object is not None else None),
@@ -1304,6 +1307,7 @@ async def _user_api_key_auth_builder(
                             else LitellmUserRoles.INTERNAL_USER
                         ),
                         user_id=user_id,
+                        user_email=user_email,
                         org_id=org_id,
                         parent_otel_span=parent_otel_span,
                         end_user_id=end_user_id,
@@ -1345,6 +1349,7 @@ async def _user_api_key_auth_builder(
                         )
                         if auto_registered is not None:
                             auto_registered.jwt_claims = jwt_claims
+                            auto_registered.user_email = user_email
                             valid_token = auto_registered
                             api_key = valid_token.token or ""
 
@@ -2607,7 +2612,7 @@ async def _return_user_api_key_auth_obj(
     start_time: datetime,
     user_role: Optional[LitellmUserRoles] = None,
 ) -> UserAPIKeyAuth:
-    end_time = datetime.now()
+    end_time = datetime.now(timezone.utc)
 
     asyncio.create_task(
         user_api_key_service_logger_obj.async_service_success_hook(
@@ -2696,9 +2701,10 @@ def _update_key_budget_with_temp_budget_increase(
 ) -> UserAPIKeyAuth:
     if valid_token.max_budget is None:
         return valid_token
-    temp_budget_increase = _get_temp_budget_increase(valid_token) or 0.0
-    valid_token.max_budget = valid_token.max_budget + temp_budget_increase
-    return valid_token
+    temp_budget_increase = _get_temp_budget_increase(valid_token)
+    if not temp_budget_increase:
+        return valid_token
+    return valid_token.model_copy(update={"max_budget": valid_token.max_budget + temp_budget_increase})
 
 
 async def _lookup_end_user_and_apply_budget(
@@ -2796,19 +2802,11 @@ async def _enforce_key_and_fallback_model_access(
                 llm_router=llm_router,
             )
 
-        # Validate every fallback model name reachable by this request.
-        # All three fields (``fallbacks``, ``context_window_fallbacks``,
-        # ``content_policy_fallbacks``) are forwarded to the router as
-        # per-request kwargs whether they appear at the top level of
-        # ``request_data`` or nested under ``router_settings_override``.
-        # Both surfaces must be validated against the API key's model
-        # allowlist or a caller can smuggle a restricted model. VERIA-44.
-        fallback_names: List[str] = []
-        override_settings = request_data.get("router_settings_override")
-        for _fb_key in ROUTER_FALLBACK_FIELDS:
-            fallback_names.extend(iter_router_fallback_model_names(request_data.get(_fb_key)))
-            if isinstance(override_settings, dict):
-                fallback_names.extend(iter_router_fallback_model_names(override_settings.get(_fb_key)))
+        fallback_names = tuple(
+            name
+            for target in iter_request_fallback_targets(request_data)
+            if (name := _fallback_target_model_name(target)) is not None
+        )
 
         for _name in dict.fromkeys(fallback_names):  # dedupe, preserve order
             await can_key_call_model(
@@ -2824,36 +2822,14 @@ async def _enforce_key_and_fallback_model_access(
             )
 
 
-ROUTER_FALLBACK_FIELDS: Tuple[str, ...] = (
-    "fallbacks",
-    "context_window_fallbacks",
-    "content_policy_fallbacks",
-)
-
-
-def iter_router_fallback_model_names(fallbacks: Any) -> Iterator[str]:
-    """Yield leaf model names from any of the supported fallbacks shapes.
-
-    Handles the simple top-level shape (``str`` or ``{"model": str}``) and
-    the nested router-config shape (``[{primary: [fallback_list]}]``).
-    """
-    if not isinstance(fallbacks, list):
-        return
-    for entry in fallbacks:
-        if isinstance(entry, str):
-            yield entry
-        elif isinstance(entry, dict):
-            if isinstance(entry.get("model"), str):
-                yield entry["model"]
-                continue
-            for fallback_list in entry.values():
-                if not isinstance(fallback_list, list):
-                    continue
-                for m in fallback_list:
-                    if isinstance(m, str):
-                        yield m
-                    elif isinstance(m, dict) and isinstance(m.get("model"), str):
-                        yield m["model"]
+def _fallback_target_model_name(target: object) -> str | None:
+    if isinstance(target, str):
+        return target
+    if isinstance(target, dict):
+        model = target.get("model")
+        if isinstance(model, str):
+            return model
+    return None
 
 
 async def _run_post_custom_auth_checks(

litellm/proxy/common_utils/callback_utils.py

@@ -1,4 +1,5 @@
 import copy
+import os
 from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Optional
 
 import litellm
@@ -564,11 +565,8 @@ def process_callback(_callback: str, callback_type: str, environment_variables:
 
     env_vars_dict: dict[str, str | None] = {}
     for _var in env_vars:
-        env_variable = environment_variables.get(_var, None)
-        if env_variable is None:
-            env_vars_dict[_var] = None
-        else:
-            env_vars_dict[_var] = env_variable
+        stored_value = environment_variables.get(_var, None)
+        env_vars_dict[_var] = stored_value if stored_value is not None else os.getenv(_var)
 
     return {"name": _callback, "variables": env_vars_dict, "type": callback_type}
 

litellm/proxy/common_utils/reset_budget_job.py

@@ -13,6 +13,11 @@ from litellm.proxy._types import (
     LiteLLM_UserTable,
     LiteLLM_VerificationToken,
 )
+from litellm.proxy.common_utils.timezone_utils import (
+    BudgetResetSettings,
+    compute_budget_reset_at,
+    get_budget_reset_settings,
+)
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.table_repositories import (
@@ -32,9 +37,15 @@ class ResetBudgetJob:
     Resets the budget for all the keys, users, and teams that need it
     """
 
-    def __init__(self, proxy_logging_obj: ProxyLogging, prisma_client: PrismaClient):
+    def __init__(
+        self,
+        proxy_logging_obj: ProxyLogging,
+        prisma_client: PrismaClient,
+        reset_settings: BudgetResetSettings | None = None,
+    ):
         self.proxy_logging_obj: ProxyLogging = proxy_logging_obj
         self.prisma_client: PrismaClient = prisma_client
+        self.reset_settings: BudgetResetSettings = reset_settings or get_budget_reset_settings()
 
     async def reset_budget(
         self,
@@ -237,7 +248,7 @@ class ResetBudgetJob:
 
             if budgets_to_reset is not None and len(budgets_to_reset) > 0:
                 for budget in budgets_to_reset:
-                    budget = await ResetBudgetJob._reset_budget_reset_at_date(budget, now)
+                    budget = await ResetBudgetJob._reset_budget_reset_at_date(budget, now, self.reset_settings)
 
                 await self.prisma_client.update_data(
                     query_type="update_many",
@@ -442,7 +453,11 @@ class ResetBudgetJob:
             if keys_to_reset is not None and len(keys_to_reset) > 0:
                 for key in keys_to_reset:
                     try:
-                        updated_key = await ResetBudgetJob._reset_budget_for_key(key=key, current_time=now)
+                        updated_key = await ResetBudgetJob._reset_budget_for_key(
+                            key=key,
+                            current_time=now,
+                            reset_settings=self.reset_settings,
+                        )
                         if updated_key is not None:
                             updated_keys.append(updated_key)
                         else:
@@ -513,7 +528,11 @@ class ResetBudgetJob:
             if users_to_reset is not None and len(users_to_reset) > 0:
                 for user in users_to_reset:
                     try:
-                        updated_user = await ResetBudgetJob._reset_budget_for_user(user=user, current_time=now)
+                        updated_user = await ResetBudgetJob._reset_budget_for_user(
+                            user=user,
+                            current_time=now,
+                            reset_settings=self.reset_settings,
+                        )
                         if updated_user is not None:
                             updated_users.append(updated_user)
                         else:
@@ -588,7 +607,11 @@ class ResetBudgetJob:
             if teams_to_reset is not None and len(teams_to_reset) > 0:
                 for team in teams_to_reset:
                     try:
-                        updated_team = await ResetBudgetJob._reset_budget_for_team(team=team, current_time=now)
+                        updated_team = await ResetBudgetJob._reset_budget_for_team(
+                            team=team,
+                            current_time=now,
+                            reset_settings=self.reset_settings,
+                        )
                         if updated_team is not None:
                             updated_teams.append(updated_team)
                         else:
@@ -655,10 +678,9 @@ class ResetBudgetJob:
         counter_key: str,
         spend_counter_cache: Any,
         now: datetime,
+        reset_settings: BudgetResetSettings,
     ) -> bool:
         """Reset a single budget window if expired. Returns True if the window was reset."""
-        from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
-
         reset_at_str = window.get("reset_at")
         if not reset_at_str:
             return False
@@ -671,7 +693,9 @@ class ResetBudgetJob:
                 await spend_counter_cache.redis_cache.async_set_cache(key=counter_key, value=0.0)
             except Exception as redis_err:
                 verbose_proxy_logger.warning("Failed to reset Redis counter %s: %s", counter_key, redis_err)
-        window["reset_at"] = get_budget_reset_time(budget_duration=window["budget_duration"]).isoformat()
+        window["reset_at"] = compute_budget_reset_at(
+            budget_duration=window["budget_duration"], settings=reset_settings
+        ).isoformat()
         return True
 
     async def reset_budget_windows(self) -> None:
@@ -703,7 +727,13 @@ class ResetBudgetJob:
                 changed = False
                 for window in windows:
                     counter_key = f"spend:key:{row['token']}:window:{window['budget_duration']}"
-                    if await ResetBudgetJob._reset_expired_window(window, counter_key, spend_counter_cache, now):
+                    if await ResetBudgetJob._reset_expired_window(
+                        window,
+                        counter_key,
+                        spend_counter_cache,
+                        now,
+                        self.reset_settings,
+                    ):
                         changed = True
                 if changed:
                     await VerificationTokenRepository(self.prisma_client).table.update(
@@ -726,7 +756,13 @@ class ResetBudgetJob:
                 changed = False
                 for window in windows:
                     counter_key = f"spend:team:{row['team_id']}:window:{window['budget_duration']}"
-                    if await ResetBudgetJob._reset_expired_window(window, counter_key, spend_counter_cache, now):
+                    if await ResetBudgetJob._reset_expired_window(
+                        window,
+                        counter_key,
+                        spend_counter_cache,
+                        now,
+                        self.reset_settings,
+                    ):
                         changed = True
                 if changed:
                     await TeamRepository(self.prisma_client).table.update(
@@ -741,6 +777,7 @@ class ResetBudgetJob:
         item: Union[LiteLLM_TeamTable, LiteLLM_UserTable, LiteLLM_VerificationToken],
         current_time: datetime,
         item_type: Literal["key", "team", "user"],
+        reset_settings: BudgetResetSettings,
     ):
         """
         In-place, updates spend=0, and sets budget_reset_at to current_time + budget_duration
@@ -755,24 +792,40 @@ class ResetBudgetJob:
         try:
             item.spend = 0.0
             if hasattr(item, "budget_duration") and item.budget_duration is not None:
-                from litellm.proxy.common_utils.timezone_utils import (
-                    get_budget_reset_time,
+                item.budget_reset_at = compute_budget_reset_at(
+                    budget_duration=item.budget_duration, settings=reset_settings
                 )
-
-                item.budget_reset_at = get_budget_reset_time(budget_duration=item.budget_duration)
             return item
         except Exception as e:
             verbose_proxy_logger.exception("Error resetting budget for %s: %s. Item: %s", item_type, e, item)
             raise e
 
     @staticmethod
-    async def _reset_budget_for_team(team: LiteLLM_TeamTable, current_time: datetime) -> Optional[LiteLLM_TeamTable]:
-        await ResetBudgetJob._reset_budget_common(item=team, current_time=current_time, item_type="team")
+    async def _reset_budget_for_team(
+        team: LiteLLM_TeamTable,
+        current_time: datetime,
+        reset_settings: BudgetResetSettings,
+    ) -> LiteLLM_TeamTable | None:
+        await ResetBudgetJob._reset_budget_common(
+            item=team,
+            current_time=current_time,
+            item_type="team",
+            reset_settings=reset_settings,
+        )
         return team
 
     @staticmethod
-    async def _reset_budget_for_user(user: LiteLLM_UserTable, current_time: datetime) -> Optional[LiteLLM_UserTable]:
-        await ResetBudgetJob._reset_budget_common(item=user, current_time=current_time, item_type="user")
+    async def _reset_budget_for_user(
+        user: LiteLLM_UserTable,
+        current_time: datetime,
+        reset_settings: BudgetResetSettings,
+    ) -> LiteLLM_UserTable | None:
+        await ResetBudgetJob._reset_budget_common(
+            item=user,
+            current_time=current_time,
+            item_type="user",
+            reset_settings=reset_settings,
+        )
         return user
 
     @staticmethod
@@ -788,15 +841,15 @@ class ResetBudgetJob:
 
     @staticmethod
     async def _reset_budget_reset_at_date(
-        budget: LiteLLM_BudgetTableFull, current_time: datetime
+        budget: LiteLLM_BudgetTableFull,
+        current_time: datetime,
+        reset_settings: BudgetResetSettings,
     ) -> LiteLLM_BudgetTableFull:
         try:
             if budget.budget_duration is not None:
-                from litellm.proxy.common_utils.timezone_utils import (
-                    get_budget_reset_time,
+                budget.budget_reset_at = compute_budget_reset_at(
+                    budget_duration=budget.budget_duration, settings=reset_settings
                 )
-
-                budget.budget_reset_at = get_budget_reset_time(budget_duration=budget.budget_duration)
         except Exception as e:
             verbose_proxy_logger.exception("Error resetting budget_reset_at for budget: %s. Item: %s", e, budget)
             raise e
@@ -804,7 +857,14 @@ class ResetBudgetJob:
 
     @staticmethod
     async def _reset_budget_for_key(
-        key: LiteLLM_VerificationToken, current_time: datetime
-    ) -> Optional[LiteLLM_VerificationToken]:
-        await ResetBudgetJob._reset_budget_common(item=key, current_time=current_time, item_type="key")
+        key: LiteLLM_VerificationToken,
+        current_time: datetime,
+        reset_settings: BudgetResetSettings,
+    ) -> LiteLLM_VerificationToken | None:
+        await ResetBudgetJob._reset_budget_common(
+            item=key,
+            current_time=current_time,
+            item_type="key",
+            reset_settings=reset_settings,
+        )
         return key

litellm/proxy/common_utils/timezone_utils.py

@@ -1,10 +1,47 @@
-from datetime import datetime, timezone
+from datetime import datetime, time, timezone
+
+from pydantic import BaseModel, ConfigDict
 
 import litellm
 from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time
 
 
-def get_budget_reset_timezone():
+class BudgetResetSettings(BaseModel):
+    """Immutable, validated settings that govern when budgets reset.
+
+    Parsed once from `litellm_settings` and injected into consumers (the reset
+    job, management endpoints) so reset times never depend on reaching into
+    module-level globals at call time.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    timezone: str = "UTC"
+    reset_time_of_day: time = time(0, 0)
+
+
+def parse_budget_reset_time(raw: object) -> time:
+    """Parse a `budget_reset_time` config value (e.g. "12:00") into a `time`.
+
+    Falls back to midnight when unset; raises a clear error on a malformed value
+    so a bad config fails loudly at startup instead of silently resetting at midnight.
+    """
+    if raw is None or raw == "":
+        return time(0, 0)
+    if not isinstance(raw, str):
+        raise ValueError(f"Invalid budget_reset_time {raw!r}; must be a quoted 24-hour 'HH:MM' string, e.g. \"12:00\"")
+    for fmt in ("%H:%M", "%H:%M:%S"):
+        try:
+            parsed = datetime.strptime(raw, fmt)
+            return time(hour=parsed.hour, minute=parsed.minute, second=parsed.second)
+        except ValueError:
+            continue
+    raise ValueError(
+        f"Invalid budget_reset_time {raw!r}; expected a 24-hour 'HH:MM' or 'HH:MM:SS' string, e.g. \"12:00\""
+    )
+
+
+def get_budget_reset_timezone() -> str:
     """
     Get the budget reset timezone from litellm_settings.
     Falls back to UTC if not specified.
@@ -15,15 +52,29 @@ def get_budget_reset_timezone():
     return getattr(litellm, "timezone", None) or "UTC"
 
 
-def get_budget_reset_time(budget_duration: str) -> datetime:
-    """
-    Get the budget reset time based on the configured timezone.
-    Falls back to UTC if not specified.
-    """
+def get_budget_reset_settings() -> BudgetResetSettings:
+    """Build validated reset settings from litellm_settings. Raises on a malformed
+    `budget_reset_time`, which lets the proxy fail fast at startup."""
+    return BudgetResetSettings(
+        timezone=get_budget_reset_timezone(),
+        reset_time_of_day=parse_budget_reset_time(getattr(litellm, "budget_reset_time", None)),
+    )
 
-    reset_at = get_next_standardized_reset_time(
+
+def compute_budget_reset_at(budget_duration: str, settings: BudgetResetSettings) -> datetime:
+    """Compute the next reset time for a budget duration using injected settings."""
+    return get_next_standardized_reset_time(
         duration=budget_duration,
         current_time=datetime.now(timezone.utc),
-        timezone_str=get_budget_reset_timezone(),
+        timezone_str=settings.timezone,
+        reset_time_of_day=settings.reset_time_of_day,
     )
-    return reset_at
+
+
+def get_budget_reset_time(budget_duration: str) -> datetime:
+    """Get the budget reset time using the globally-configured timezone and reset time.
+
+    Thin wrapper over `compute_budget_reset_at` for callers that don't yet receive
+    `BudgetResetSettings` by injection (creation/update endpoints, startup backfill).
+    """
+    return compute_budget_reset_at(budget_duration, get_budget_reset_settings())

litellm/proxy/guardrails/guardrail_hooks/model_armor/__init__.py

@@ -27,6 +27,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         mask_response_content=litellm_params.mask_response_content,
         fail_on_error=litellm_params.fail_on_error,
         skip_unscannable_attachments=litellm_params.skip_unscannable_attachments,
+        sanitize_error_detail=litellm_params.sanitize_error_detail,
     )
     litellm.logging_callback_manager.add_litellm_callback(_model_armor_callback)
 

litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py

@@ -11,6 +11,7 @@ from typing import (
     Union,
 )
 
+import httpx
 from fastapi import HTTPException
 
 if TYPE_CHECKING:
@@ -35,7 +36,8 @@ from litellm.proxy.guardrails.guardrail_hooks.model_armor.file_scanning import (
     MODEL_ARMOR_MAX_FILE_SIZE_BYTES,
     plan_file_scans,
 )
-from litellm.types.guardrails import GuardrailEventHooks
+from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
+from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import (
     CallTypes,
@@ -50,6 +52,33 @@ from litellm.types.utils import (
 GUARDRAIL_NAME = "model_armor"
 
 
+class ModelArmorAPIError(Exception):
+    """Model Armor API failure (non-2xx), distinct from a content-block decision so
+    hooks can honor fail_on_error. The detail is already sanitized per configuration."""
+
+    def __init__(self, detail: str):
+        super().__init__(detail)
+        self.detail = detail
+
+
+_SCANNED_CONTENT_KEYS = frozenset({"text", "sanitizedText", "findings", "maliciousUriMatchedItems"})
+
+RedactablePayload = Union[dict, list, str, int, float, bool, None]
+
+
+def _redact_scanned_content(payload: RedactablePayload, depth: int = 0) -> RedactablePayload:
+    if depth >= DEFAULT_MAX_RECURSE_DEPTH:
+        return "[REDACTED]"
+    if isinstance(payload, dict):
+        return {
+            key: "[REDACTED]" if key in _SCANNED_CONTENT_KEYS else _redact_scanned_content(value, depth + 1)
+            for key, value in payload.items()
+        }
+    if isinstance(payload, list):
+        return [_redact_scanned_content(item, depth + 1) for item in payload]
+    return payload
+
+
 class ModelArmorGuardrail(CustomGuardrail, VertexBase):
     """
     Google Cloud Model Armor Guardrail integration for LiteLLM.
@@ -76,6 +105,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         location: Optional[str] = None,
         credentials: Optional[Any] = None,
         api_endpoint: Optional[str] = None,
+        sanitize_error_detail: "bool | None" = True,
         **kwargs,
     ):
         # Set supported event hooks if not already provided
@@ -98,6 +128,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         self.location = location or "us-central1"
         self.credentials = credentials
         self.api_endpoint = api_endpoint
+        self.sanitize_error_detail = sanitize_error_detail is not False
 
         # Store optional params
         self.optional_params = kwargs
@@ -141,6 +172,67 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         verbose_proxy_logger.debug("Model Armor: Skipping non-ModelResponse type: %s", type(response).__name__)
         return ""
 
+    def _build_api_error_detail(self, status_code: int, response_text: str) -> str:
+        if self.sanitize_error_detail:
+            return f"Model Armor API error (upstream {status_code})"
+        return f"Model Armor API error (upstream {status_code}): {response_text}"
+
+    def _build_block_error_detail(self, message: str, armor_response: RedactablePayload) -> dict:
+        if self.sanitize_error_detail:
+            return {"error": message}
+        return {"error": message, "model_armor_response": armor_response}
+
+    def _build_logging_response(self, armor_response: RedactablePayload) -> RedactablePayload:
+        if self.sanitize_error_detail:
+            return _redact_scanned_content(armor_response)
+        return armor_response
+
+    def _raise_if_fail_closed(self, e: ModelArmorAPIError) -> None:
+        if self.optional_params.get("fail_on_error", True):
+            raise e from None
+
+    def update_in_memory_litellm_params(self, litellm_params: LitellmParams) -> None:
+        super().update_in_memory_litellm_params(litellm_params)
+        self.sanitize_error_detail = self.sanitize_error_detail is not False
+
+    def _log_request_debug(
+        self,
+        url: str,
+        body: dict,
+        file_bytes: "bytes | None",
+        file_type: "str | None",
+    ) -> None:
+        # Never log byteData: it is the full base64 of the scanned document. Log only its
+        # type and size so debug deployments cannot leak the contents the guardrail inspects.
+        if file_bytes is not None and file_type is not None:
+            verbose_proxy_logger.debug(
+                "Model Armor file request - URL: %s, byteDataType: %s, bytes: %d",
+                url,
+                file_type,
+                len(file_bytes),
+            )
+        elif self.sanitize_error_detail:
+            verbose_proxy_logger.debug("Model Armor request - URL: %s", url)
+        else:
+            verbose_proxy_logger.debug(
+                "Model Armor request - URL: %s, Body: %s",
+                url,
+                body,
+            )
+
+    def _log_response_debug(self, status_code: int, response_text: str) -> None:
+        if self.sanitize_error_detail:
+            verbose_proxy_logger.debug(
+                "Model Armor response - Status: %s",
+                status_code,
+            )
+        else:
+            verbose_proxy_logger.debug(
+                "Model Armor response - Status: %s, Body: %s",
+                status_code,
+                response_text,
+            )
+
     async def make_model_armor_request(
         self,
         content: Optional[str] = None,
@@ -185,48 +277,37 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             "Authorization": f"Bearer {access_token}",
         }
 
-        # Never log byteData: it is the full base64 of the scanned document. Log only its
-        # type and size so debug deployments cannot leak the contents the guardrail inspects.
-        if file_bytes is not None and file_type is not None:
-            verbose_proxy_logger.debug(
-                "Model Armor file request - URL: %s, byteDataType: %s, bytes: %d",
-                url,
-                file_type,
-                len(file_bytes),
-            )
-        else:
-            verbose_proxy_logger.debug(
-                "Model Armor request - URL: %s, Body: %s",
-                url,
-                body,
-            )
+        self._log_request_debug(url=url, body=body, file_bytes=file_bytes, file_type=file_type)
 
         # Make request
         if self.async_handler is None:
             raise ValueError("Async handler not initialized")
 
-        response = await self.async_handler.post(
-            url=url,
-            json=body,
-            headers=headers,
-        )
+        try:
+            response = await self.async_handler.post(
+                url=url,
+                json=body,
+                headers=headers,
+            )
+        except httpx.HTTPStatusError as e:
+            detail = self._build_api_error_detail(e.response.status_code, e.response.text)
+            verbose_proxy_logger.error(
+                "Model Armor API error - Status: %s, Detail: %s",
+                e.response.status_code,
+                detail,
+            )
+            raise ModelArmorAPIError(detail) from None
 
-        verbose_proxy_logger.debug(
-            "Model Armor response - Status: %s, Body: %s",
-            response.status_code,
-            response.text,
-        )
+        self._log_response_debug(status_code=response.status_code, response_text=response.text)
 
         if response.status_code != 200:
+            detail = self._build_api_error_detail(response.status_code, response.text)
             verbose_proxy_logger.error(
-                "Model Armor API error - Status: %s, Response: %s",
+                "Model Armor API error - Status: %s, Detail: %s",
                 response.status_code,
-                response.text,
-            )
-            raise HTTPException(
-                status_code=400,
-                detail=f"Model Armor API error (upstream {response.status_code}): {response.text}",
+                detail,
             )
+            raise ModelArmorAPIError(detail)
 
         json_response = response.json()
         if hasattr(json_response, "__await__"):
@@ -351,9 +432,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
         Override to store only the Model Armor API response, not the entire data dict.
         This prevents circular references in logging.
         """
-        # Retrieve the Model Armor response & status stored on the per-request `metadata` object.
         metadata = request_data.get("metadata", {}) if isinstance(request_data, dict) else {}
-
         guardrail_response = metadata.get("_model_armor_response", {})
 
         # Determine status – default to "success" but prefer the explicit value if present.
@@ -444,6 +523,9 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                     file_bytes=attachment.file_bytes,
                     file_type=attachment.byte_data_type,
                 )
+            except ModelArmorAPIError as e:
+                self._raise_if_fail_closed(e)
+                continue
             except HTTPException:
                 raise
             except Exception as e:
@@ -459,7 +541,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             # otherwise a PII-only (SDP deidentify) document would pass through unscrubbed.
             blocked = self._should_block_content(armor_response, allow_sanitization=False)
             metadata["_model_armor_response"] = self._append_armor_response(
-                metadata.get("_model_armor_response"), armor_response
+                metadata.get("_model_armor_response"),
+                self._build_logging_response(armor_response),
             )
             if blocked or metadata.get("_model_armor_status") == "blocked":
                 metadata["_model_armor_status"] = "blocked"
@@ -469,10 +552,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             if blocked:
                 raise HTTPException(
                     status_code=400,
-                    detail={
-                        "error": "Content blocked by Model Armor",
-                        "model_armor_response": armor_response,
-                    },
+                    detail=self._build_block_error_detail("Content blocked by Model Armor", armor_response),
                 )
 
     @log_guardrail_information
@@ -530,7 +610,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                 metadata = data.setdefault("metadata", {})  # ensures metadata exists and is unique per request
                 # Accumulate so a prior file scan on the same request is not overwritten by this text scan.
                 metadata["_model_armor_response"] = self._append_armor_response(
-                    metadata.get("_model_armor_response"), armor_response
+                    metadata.get("_model_armor_response"),
+                    self._build_logging_response(armor_response),
                 )
                 # Pre-compute guardrail status for downstream logging. A blocked response will eventually raise
                 #   an HTTPException, however in scenarios where the caller decides to ignore the exception (e.g.
@@ -548,10 +629,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             if blocked:
                 raise HTTPException(
                     status_code=400,
-                    detail={
-                        "error": "Content blocked by Model Armor",
-                        "model_armor_response": armor_response,
-                    },
+                    detail=self._build_block_error_detail("Content blocked by Model Armor", armor_response),
                 )
 
             # If mask_request_content is enabled, update messages with sanitized content
@@ -565,6 +643,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
 
                     data["messages"] = set_last_user_message(messages, sanitized_content)
 
+        except ModelArmorAPIError as e:
+            self._raise_if_fail_closed(e)
         except HTTPException:
             raise
         except Exception as e:
@@ -625,7 +705,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                 metadata = data.setdefault("metadata", {})
                 # Accumulate so a prior file scan on the same request is not overwritten by this text scan.
                 metadata["_model_armor_response"] = self._append_armor_response(
-                    metadata.get("_model_armor_response"), armor_response
+                    metadata.get("_model_armor_response"),
+                    self._build_logging_response(armor_response),
                 )
                 if blocked or metadata.get("_model_armor_status") == "blocked":
                     metadata["_model_armor_status"] = "blocked"
@@ -640,10 +721,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             if blocked:
                 raise HTTPException(
                     status_code=400,
-                    detail={
-                        "error": "Content blocked by Model Armor",
-                        "model_armor_response": armor_response,
-                    },
+                    detail=self._build_block_error_detail("Content blocked by Model Armor", armor_response),
                 )
 
             # If mask_request_content is enabled, update messages with sanitized content
@@ -656,6 +734,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
 
                     data["messages"] = set_last_user_message(messages, sanitized_content)
 
+        except ModelArmorAPIError as e:
+            self._raise_if_fail_closed(e)
         except HTTPException:
             raise
         except Exception as e:
@@ -698,7 +778,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             # Attach Model Armor response & status to this request's metadata to prevent race conditions
             if isinstance(armor_response, dict):
                 model_armor_logged_object = {
-                    "model_armor_response": armor_response,
+                    "model_armor_response": self._build_logging_response(armor_response),
                     "model_armor_status": (
                         "blocked"
                         if self._should_block_content(
@@ -729,10 +809,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content):
                 raise HTTPException(
                     status_code=400,
-                    detail={
-                        "error": "Response blocked by Model Armor",
-                        "model_armor_response": armor_response,
-                    },
+                    detail=self._build_block_error_detail("Response blocked by Model Armor", armor_response),
                 )
 
             # If mask_response_content is enabled, update response with sanitized content
@@ -746,6 +823,8 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                                 if choice.message.content:
                                     choice.message.content = sanitized_content
 
+        except ModelArmorAPIError as e:
+            self._raise_if_fail_closed(e)
         except HTTPException:
             raise
         except Exception as e:
@@ -790,7 +869,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                     # Attach Model Armor response & status to this request's metadata to avoid race conditions
                     if isinstance(request_data, dict):
                         metadata = request_data.setdefault("metadata", {})
-                        metadata["_model_armor_response"] = armor_response
+                        metadata["_model_armor_response"] = self._build_logging_response(armor_response)
                         metadata["_model_armor_status"] = (
                             "blocked" if self._should_block_content(armor_response) else "success"
                         )
@@ -809,10 +888,10 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                     if self._should_block_content(armor_response):
                         raise HTTPException(
                             status_code=400,
-                            detail={
-                                "error": "Streaming response blocked by Model Armor",
-                                "model_armor_response": armor_response,
-                            },
+                            detail=self._build_block_error_detail(
+                                "Streaming response blocked by Model Armor",
+                                armor_response,
+                            ),
                         )
 
                     # Apply sanitization if enabled
@@ -831,6 +910,11 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                                 yield chunk
                             return
 
+                except ModelArmorAPIError as e:
+                    if self.optional_params.get("fail_on_error", True):
+                        error_obj = {"message": e.detail, "code": "500"}
+                        yield f"data: {json.dumps({'error': error_obj})}\n\n"
+                        return
                 except HTTPException as e:
                     # Yield error as SSE event so create_response() detects it and
                     # returns a proper JSON error response with the correct status code.

litellm/proxy/litellm_pre_call_utils.py

@@ -19,7 +19,10 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     iter_client_callback_metadata_dicts,
 )
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.url_utils import is_url_destination_allowed_by_host
+from litellm.litellm_core_utils.url_utils import (
+    is_url_destination_allowed_by_host,
+    provider_url_destination_candidates,
+)
 from litellm.proxy._types import (
     AddTeamCallback,
     CommonProxyErrors,
@@ -227,23 +230,26 @@ def _reject_url_valued_destinations(data: Dict[str, Any]) -> None:
     allowed_hosts = getattr(litellm, "provider_url_destination_allowed_hosts", []) or []
     for field in _URL_DESTINATION_REQUEST_FIELDS:
         value = data.get(field)
-        if not isinstance(value, str) or not value.startswith(("http://", "https://")):
+        if not isinstance(value, str):
             continue
-        if is_url_destination_allowed_by_host(value, allowed_hosts):
-            continue
-        raise HTTPException(
-            status_code=400,
-            detail={
-                "error": "invalid_request",
-                "param": field,
-                "message": (
-                    f"URL-valued '{field}' is not allowed. Configure custom "
-                    "endpoints with api_base instead, or add the destination "
-                    "host to `provider_url_destination_allowed_hosts` in "
-                    "litellm_settings."
-                ),
-            },
-        )
+        for candidate in provider_url_destination_candidates(value):
+            if not candidate.lower().startswith(("http://", "https://")):
+                continue
+            if is_url_destination_allowed_by_host(candidate, allowed_hosts):
+                continue
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": "invalid_request",
+                    "param": field,
+                    "message": (
+                        f"URL-valued '{field}' is not allowed. Configure custom "
+                        "endpoints with api_base instead, or add the destination "
+                        "host to `provider_url_destination_allowed_hosts` in "
+                        "litellm_settings."
+                    ),
+                },
+            )
 
 
 def _strip_untrusted_request_header_controls(

litellm/proxy/management_endpoints/cache_settings_endpoints.py

@@ -18,8 +18,9 @@ from pydantic import BaseModel, Field
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm._redis import _redis_kwargs_from_environment
 from litellm._uuid import uuid
-from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_keys
+from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 from litellm.proxy._types import (
     AUDIT_ACTIONS,
     LiteLLM_AuditLogs,
@@ -43,6 +44,17 @@ router = APIRouter()
 # (e.g. redis://:secret@host:6379/1).
 _CACHE_SENSITIVE_FIELDS: set = {"password", "sentinel_password", "url"}
 
+# The env fallback resolves the full set of redis.Redis kwargs, which includes
+# credential-bearing params (azure_client_secret, ssl_password, ...) that are
+# not cache UI fields. Only overlay fields the settings page actually renders,
+# so the read never surfaces a credential the UI does not manage.
+_CACHE_SETTINGS_FIELD_NAMES: frozenset = frozenset(field.field_name for field in CACHE_SETTINGS_FIELDS)
+
+# Classifier used, alongside _CACHE_SENSITIVE_FIELDS, to redact any
+# credential-bearing key before it leaves the server (`url` is kept in the
+# explicit set because its name carries no sensitive segment).
+_CREDENTIAL_CLASSIFIER = SensitiveDataMasker()
+
 
 _REDACTED_VALUE = "***REDACTED***"
 
@@ -67,6 +79,165 @@ def _resolve_cache_url_precedence(settings: Mapping[str, Any]) -> dict[str, Any]
     return {k: v for k, v in settings.items() if k not in _URL_OVERRIDDEN_CONNECTION_FIELDS}
 
 
+def _parse_stored_settings(cache_settings_value: object) -> dict[str, Any]:
+    """Normalize a stored cache_settings blob to a dict.
+
+    The prisma column comes back as either a JSON string or an already-parsed
+    dict depending on the client, so callers that json.loads unconditionally
+    silently drop the whole (still-encrypted) row on the dict path.
+    """
+    parsed = json.loads(cache_settings_value) if isinstance(cache_settings_value, str) else cache_settings_value
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def _overlay_environment(stored: Mapping[str, Any]) -> dict[str, Any]:
+    """Fill connection fields from the REDIS_* environment the cache actually reads.
+
+    A response cache pointed at Redis resolves host/port/password/etc. from the
+    REDIS_* env vars when the stored config leaves them unset, so a cache
+    configured purely through the environment works while its settings page,
+    which reads only the database row, shows blank. Overlaying the same env
+    kwargs the runtime uses makes the page reflect the effective connection.
+    Stored values win; the environment only fills what the stored config omits.
+    """
+    env_kwargs = {
+        key: value for key, value in _redis_kwargs_from_environment().items() if key in _CACHE_SETTINGS_FIELD_NAMES
+    }
+    if not env_kwargs:
+        return dict(stored)
+    effective = {**env_kwargs, **stored}
+    # the env fallback is a Redis connection, so name the type when the stored
+    # config did not, letting the UI render the Redis fields it just populated
+    effective.setdefault("type", "redis")
+    return effective
+
+
+def _redact_credentials(settings: Mapping[str, Any]) -> dict[str, Any]:
+    """Replace credential-bearing values with a fixed marker, keeping the rest.
+
+    The marker is unambiguous on the way back in: an admin who edits an
+    unrelated field and re-submits sends the marker for the untouched secret,
+    which the update path maps back to the stored value rather than persisting
+    the marker over a working password.
+    """
+    return {
+        key: (_REDACTED_VALUE if value is not None and _is_credential_field(key) else value)
+        for key, value in settings.items()
+    }
+
+
+def _is_credential_field(key: str) -> bool:
+    """Whether a cache setting carries a credential and must be redacted on read."""
+    return key in _CACHE_SENSITIVE_FIELDS or _CREDENTIAL_CLASSIFIER.is_sensitive_key(key)
+
+
+def _has_connection_target(value: object) -> bool:
+    """Whether a payload value names a live discrete connection target."""
+    if isinstance(value, str):
+        return value.strip() != "" and value != _REDACTED_VALUE
+    return value not in (None, [], {})
+
+
+# Every field that identifies which Redis a credential belongs to, across node
+# (host/port/url), cluster (redis_startup_nodes), and sentinel
+# (sentinel_nodes/service_name) modes. A stored secret is bound to these.
+_CONNECTION_TARGET_FIELDS: tuple = (
+    "host",
+    "port",
+    "url",
+    "redis_startup_nodes",
+    "sentinel_nodes",
+    "service_name",
+)
+
+
+def _target_repr(value: object) -> str:
+    """Canonical string form of a connection-target value for equality checks.
+
+    The client may serialize the same target differently from storage (a port as
+    "6379" vs 6379, node lists round-tripped through JSON), so compare normalized
+    forms rather than raw values to avoid treating an unchanged target as a change.
+    """
+    if isinstance(value, (list, dict)):
+        return json.dumps(value, sort_keys=True, default=str)
+    return str(value)
+
+
+def _saved_secret_is_reusable(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> bool:
+    """Whether a stored credential may be restored for this request.
+
+    A stored secret belongs to the stored connection target, so it is reused only
+    when the request describes that same target on every dimension the stored
+    config pins (host/port, url, cluster nodes, sentinel nodes/service). This
+    prevents credential replay: a caller cannot omit the credential, point at a
+    different (or incomplete) target, and have the proxy send the stored secret
+    to a Redis of their choosing.
+
+    Non-secret target fields (host/port/nodes/service) must be supplied and match
+    in normalized form, so equivalent representations (port "6379" vs 6379) are
+    not seen as a change while an omitted or different value is. ``url`` is the
+    exception: it is itself the secret and the form never re-prefills it, so a
+    redacted or omitted url means "keep the stored url" (same target) and only a
+    different supplied url blocks reuse.
+    """
+    for field in _CONNECTION_TARGET_FIELDS:
+        saved_value = saved.get(field)
+        if saved_value in (None, "", [], {}):
+            continue  # the stored config does not pin this dimension
+        incoming_value = incoming.get(field)
+        if field == "url":
+            if incoming_value in (None, "", _REDACTED_VALUE):
+                continue  # url kept as-is (same target)
+            if _target_repr(incoming_value) != _target_repr(saved_value):
+                return False
+            continue
+        if _target_repr(incoming_value) != _target_repr(saved_value):
+            return False  # a pinned target field is missing or different
+    return True
+
+
+def _merge_over_saved(incoming: Mapping[str, Any], saved: Mapping[str, Any]) -> dict[str, Any]:
+    """Keep the stored secret behind any credential the caller echoed back redacted or omitted.
+
+    GET returns credentials as the marker and the form never re-prefills a
+    secret, so a save that does not touch a credential arrives with the marker
+    or with the field absent. Either way the real secret must survive: it is
+    restored from the stored row, or dropped when there is no stored row (the
+    value is env-sourced and the marker must never be persisted). Non-secret
+    fields are taken from the incoming payload as-is, so clearing one still works.
+
+    ``url`` is the exception: it is credential-bearing (redacted) yet also a
+    connection-mode selector that url-precedence resolves against host/port. If
+    the caller supplies a discrete target (host, cluster, or sentinel nodes), a
+    stored url is a stale mode the caller is leaving, so it is dropped rather
+    than restored, otherwise url-precedence would resurrect it and discard the
+    submitted host/port.
+    """
+    switching_to_discrete_target = (
+        _has_connection_target(incoming.get("host"))
+        or _has_connection_target(incoming.get("redis_startup_nodes"))
+        or _has_connection_target(incoming.get("sentinel_nodes"))
+    )
+    reuse_saved_secret = _saved_secret_is_reusable(incoming, saved)
+    merged = dict(incoming)
+    for field in _CACHE_SENSITIVE_FIELDS:
+        # A value the caller explicitly supplied is honored verbatim: a new
+        # secret, or an empty string / null to clear the stored one. Only an
+        # omitted field or the echoed-back marker triggers preserve-or-drop.
+        if field in incoming and incoming[field] != _REDACTED_VALUE:
+            continue
+        if field == "url" and switching_to_discrete_target:
+            merged.pop(field, None)
+            continue
+        if field in saved and reuse_saved_secret:
+            merged[field] = saved[field]
+        else:
+            # nothing stored to reuse, or the caller is pointing at a different
+            # target: never persist/replay the marker or the stored secret
+            merged.pop(field, None)
+    return merged
+
+
 def _redact_settings(settings: Optional[Mapping[str, Any]]) -> Dict[str, Any]:
     """Replace every value in a settings map with a fixed marker.
 
@@ -270,34 +441,34 @@ async def get_cache_settings(
         # Get cache settings fields from types file
         cache_fields = [field.model_copy(deep=True) for field in CACHE_SETTINGS_FIELDS]
 
-        # Try to get cache settings from database
-        current_values = {}
+        # Read the stored settings (decrypted); an env-only cache has none.
+        stored: dict[str, Any] = {}
         if prisma_client is not None:
             cache_config = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"})
             if cache_config is not None and cache_config.cache_settings:
-                # Decrypt cache settings
-                cache_settings_json = cache_config.cache_settings
-                if isinstance(cache_settings_json, str):
-                    cache_settings_dict = json.loads(cache_settings_json)
-                else:
-                    cache_settings_dict = cache_settings_json
+                stored = proxy_config._decrypt_db_variables(
+                    variables_dict=_parse_stored_settings(cache_config.cache_settings)
+                )
 
-                # Decrypt environment variables
-                decrypted_settings = proxy_config._decrypt_db_variables(variables_dict=cache_settings_dict)
+        # Fill connection fields from the REDIS_* environment the cache resolves
+        # from when the stored config leaves them unset, then apply url precedence
+        # so a url-mode config does not surface conflicting discrete fields (which
+        # would otherwise let a no-op save silently switch it to host/port).
+        effective = _resolve_cache_url_precedence(_overlay_environment(stored))
 
-                # Derive redis_type for UI based on settings
-                # UI uses redis_type to show/hide fields, backend only stores 'type'
-                if decrypted_settings.get("type") == "redis":
-                    if decrypted_settings.get("redis_startup_nodes"):
-                        decrypted_settings["redis_type"] = "cluster"
-                    elif decrypted_settings.get("sentinel_nodes"):
-                        decrypted_settings["redis_type"] = "sentinel"
-                    else:
-                        decrypted_settings["redis_type"] = "node"
+        # Derive redis_type for UI based on settings
+        # UI uses redis_type to show/hide fields, backend only stores 'type'
+        if effective.get("type") == "redis":
+            if effective.get("redis_startup_nodes"):
+                effective["redis_type"] = "cluster"
+            elif effective.get("sentinel_nodes"):
+                effective["redis_type"] = "sentinel"
+            else:
+                effective["redis_type"] = "node"
 
-                # Mask credential fields so the GET response never carries
-                # plaintext Redis / Sentinel passwords off the server.
-                current_values = mask_sensitive_keys(decrypted_settings, _CACHE_SENSITIVE_FIELDS)
+        # Redact credential fields so the GET response never carries a plaintext
+        # Redis / Sentinel password off the server.
+        current_values = _redact_credentials(effective)
 
         # Update field values with current values
         for field in cache_fields:
@@ -331,10 +502,27 @@ async def test_cache_connection(
     to verify the credentials work without affecting global state.
     """
     from litellm import Cache
+    from litellm.proxy.proxy_server import prisma_client, proxy_config
 
     try:
-        cache_settings = _resolve_cache_url_precedence(request.cache_settings)
-        verbose_proxy_logger.debug("Testing cache connection with settings: %s", cache_settings)
+        # A credential the form left untouched arrives redacted; resolve it back
+        # to the stored secret so the test connects with the real password. A
+        # lookup failure must not block the test, so fall back to no stored row.
+        saved_settings: dict[str, Any] = {}
+        if prisma_client is not None:
+            try:
+                existing_row = await CacheConfigRepository(prisma_client).table.find_unique(
+                    where={"id": "cache_config"}
+                )
+                if existing_row is not None and existing_row.cache_settings:
+                    saved_settings = proxy_config._decrypt_db_variables(
+                        variables_dict=_parse_stored_settings(existing_row.cache_settings)
+                    )
+            except Exception:  # noqa: BLE001 - a saved-settings lookup failure must not block a connection test
+                saved_settings = {}
+        cache_settings = _resolve_cache_url_precedence(_merge_over_saved(request.cache_settings, saved_settings))
+        # cache_settings now carries the resolved plaintext credential; never log it raw
+        verbose_proxy_logger.debug("Testing cache connection with settings: %s", _redact_credentials(cache_settings))
 
         # Only support Redis for now
         if cache_settings.get("type") != "redis":
@@ -400,19 +588,20 @@ async def update_cache_settings(
         )
 
     try:
-        cache_settings = _resolve_cache_url_precedence(request.cache_settings)
-
-        # Snapshot the prior settings (key set only — values get redacted in
-        # the audit row) so the audit-log entry shows which fields changed.
+        # Read the stored row first: its decrypted values back any credential the
+        # caller echoed back redacted, and its key set drives the audit diff.
         existing_row = await CacheConfigRepository(prisma_client).table.find_unique(where={"id": "cache_config"})
         before_settings: Optional[Dict[str, Any]] = None
+        saved_settings: dict[str, Any] = {}
         if existing_row is not None and existing_row.cache_settings:
-            try:
-                before_settings = json.loads(existing_row.cache_settings)
-            except (TypeError, ValueError):
-                before_settings = None
+            before_settings = _parse_stored_settings(existing_row.cache_settings)
+            saved_settings = proxy_config._decrypt_db_variables(variables_dict=before_settings)
         action: AUDIT_ACTIONS = "updated" if existing_row is not None else "created"
 
+        # Preserve stored secrets behind any redacted or omitted credential, then
+        # resolve the url-vs-discrete-fields precedence.
+        cache_settings = _resolve_cache_url_precedence(_merge_over_saved(request.cache_settings, saved_settings))
+
         # Encrypt sensitive fields (keep redis_type for storage)
         encrypted_settings = proxy_config._encrypt_env_variables(environment_variables=cache_settings)
 
@@ -461,7 +650,7 @@ async def update_cache_settings(
         return {
             "message": "Cache settings updated successfully",
             "status": "success",
-            "settings": cache_settings,
+            "settings": _redact_credentials(cache_settings),
         }
     except Exception as e:
         verbose_proxy_logger.error(f"Error updating cache settings: {str(e)}")

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -42,6 +42,9 @@ from litellm.proxy._experimental.mcp_server.db import (
     rotate_mcp_user_credentials_master_key,
     rotate_mcp_user_env_vars_master_key,
 )
+from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import (
+    rotate_sso_identity_assertions_master_key,
+)
 from litellm.proxy._types import *
 from litellm.proxy._types import LiteLLM_VerificationToken, hash_token
 from litellm.proxy.auth.auth_checks import (
@@ -4242,6 +4245,15 @@ async def _rotate_master_key(
     except Exception as e:
         verbose_proxy_logger.warning("Failed to rotate MCP user env vars: %s", str(e))
 
+    # 4d. process SSO identity assertion table (EMA subject tokens)
+    try:
+        await rotate_sso_identity_assertions_master_key(
+            prisma_client=prisma_client,
+            new_master_key=new_master_key,
+        )
+    except Exception as e:  # noqa: BLE001  # one store's failure must not abort the master-key rotation
+        verbose_proxy_logger.warning("Failed to rotate SSO identity assertions: %s", str(e))
+
     # 5. process credentials table
     try:
         credentials = await CredentialsRepository(prisma_client).table.find_many()

litellm/proxy/management_endpoints/ui_sso.py

@@ -62,6 +62,11 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_store import (
+    SSOIdentityAssertion,
+    assertion_from_sso_login,
+    retain_sso_identity_assertion_for_ema,
+)
 from litellm.proxy._types import (
     CommonProxyErrors,
     LiteLLM_UserTable,
@@ -1311,12 +1316,15 @@ async def get_generic_sso_response(
     sso_jwt_handler: Optional[JWTHandler],  # sso specific jwt handler - used for restricted sso group access control
     generic_client_id: str,
     redirect_url: str,
-) -> Tuple[Union[OpenID, dict], Optional[dict], Optional[dict]]:  # (result, received_response, access_token_payload)
+) -> tuple[
+    Union[OpenID, dict], dict | None, dict | None, SSOIdentityAssertion | None
+]:  # (result, received_response, access_token_payload, sso_assertion)
     # make generic sso provider
     from fastapi_sso.sso.base import DiscoveryDocument
     from fastapi_sso.sso.generic import create_provider
 
     received_response: Optional[dict] = None
+    sso_assertion: SSOIdentityAssertion | None = None
 
     # Setup environment variables
     (
@@ -1450,6 +1458,9 @@ async def get_generic_sso_response(
             # Assign directly rather than relying on nonlocal mutation so that Pyright
             # can track that received_response is non-None from this point on.
             received_response = {k: v for k, v in combined_response.items() if k not in _OAUTH_TOKEN_FIELDS}
+            sso_assertion = assertion_from_sso_login(
+                combined_response.get("id_token"), combined_response.get("refresh_token")
+            )
             # In the PKCE path verify_and_process is skipped, so generic_sso.access_token
             # is never set. Read the token directly from the exchange response instead so
             # process_sso_jwt_access_token can extract JWT-embedded roles/teams.
@@ -1461,6 +1472,7 @@ async def get_generic_sso_response(
                 headers=additional_generic_sso_headers_dict,
             )
             access_token_str = generic_sso.access_token
+            sso_assertion = assertion_from_sso_login(generic_sso.id_token, generic_sso.refresh_token)
 
         access_token_payload = process_sso_jwt_access_token(
             access_token_str, sso_jwt_handler, result, role_mappings=role_mappings
@@ -1480,7 +1492,7 @@ async def get_generic_sso_response(
             additional_generic_sso_headers_dict,
         )
     verbose_proxy_logger.debug("generic result: %s", result)
-    return result or {}, received_response, access_token_payload
+    return result or {}, received_response, access_token_payload, sso_assertion
 
 
 async def create_team_member_add_task(team_id, user_info):
@@ -1812,6 +1824,7 @@ async def auth_callback(request: Request, state: Optional[str] = None):
     generic_client_id = os.getenv("GENERIC_CLIENT_ID", None)
     received_response: Optional[dict] = None
     access_token_payload: Optional[dict] = None
+    sso_assertion: SSOIdentityAssertion | None = None
     # get url from request
     if master_key is None:
         raise ProxyException(
@@ -1842,6 +1855,7 @@ async def auth_callback(request: Request, state: Optional[str] = None):
             result,
             received_response,
             access_token_payload,
+            sso_assertion,
         ) = await get_generic_sso_response(
             request=request,
             jwt_handler=jwt_handler,
@@ -1869,6 +1883,7 @@ async def auth_callback(request: Request, state: Optional[str] = None):
             prefill_user_code=prefill_user_code,
             result=result,
             received_response=received_response,
+            sso_assertion=sso_assertion,
         )
 
     # Control-plane cross-origin: read return_to from cookie.
@@ -1884,6 +1899,7 @@ async def auth_callback(request: Request, state: Optional[str] = None):
         access_token_payload=access_token_payload,
         jwt_handler=jwt_handler,
         return_to=cp_return_to,
+        sso_assertion=sso_assertion,
     )
 
 
@@ -1943,6 +1959,7 @@ async def _complete_cli_sso_callback_session(
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
     prefill_user_code: str | None = None,
+    sso_assertion: SSOIdentityAssertion | None = None,
 ):
     from fastapi.responses import HTMLResponse
 
@@ -1962,6 +1979,8 @@ async def _complete_cli_sso_callback_session(
     if not user_info.user_id:
         raise HTTPException(status_code=500, detail="Failed to retrieve user information from SSO")
 
+    await retain_sso_identity_assertion_for_ema(user_id=user_info.user_id, assertion=sso_assertion)
+
     teams: List[str] = []
     if hasattr(user_info, "teams") and user_info.teams:
         teams = user_info.teams if isinstance(user_info.teams, list) else []
@@ -2012,6 +2031,7 @@ async def cli_sso_callback(
     result: Optional[Union[OpenID, dict]] = None,
     received_response: Optional[dict] = None,
     prefill_user_code: str | None = None,
+    sso_assertion: SSOIdentityAssertion | None = None,
 ):
     """CLI SSO callback - stores session info for JWT generation on polling"""
     verbose_proxy_logger.info("CLI SSO callback")
@@ -2065,6 +2085,7 @@ async def cli_sso_callback(
             user_api_key_cache=user_api_key_cache,
             proxy_logging_obj=proxy_logging_obj,
             prefill_user_code=prefill_user_code,
+            sso_assertion=sso_assertion,
         )
     except ProxyException:
         raise
@@ -3018,6 +3039,7 @@ class SSOAuthenticationHandler:
         access_token_payload: Optional[dict] = None,
         jwt_handler: Optional[JWTHandler] = None,
         return_to: Optional[str] = None,
+        sso_assertion: SSOIdentityAssertion | None = None,
     ) -> RedirectResponse:
         import jwt
 
@@ -3148,6 +3170,9 @@ class SSOAuthenticationHandler:
                     },
                 )
 
+        if isinstance(user_id, str) and user_id:
+            await retain_sso_identity_assertion_for_ema(user_id=user_id, assertion=sso_assertion)
+
         disabled_non_admin_personal_key_creation = get_disabled_non_admin_personal_key_creation()
         litellm_dashboard_ui = get_custom_url(request_base_url=str(request.base_url), route="ui/")
 
@@ -4241,6 +4266,7 @@ async def debug_sso_callback(request: Request):
             result,
             received_response,
             access_token_payload,
+            _sso_assertion,
         ) = await get_generic_sso_response(
             request=request,
             jwt_handler=jwt_handler,

litellm/proxy/proxy_server.py

@@ -236,6 +236,7 @@ from litellm.constants import (
     PROXY_BATCH_WRITE_AT,
     PROXY_BUDGET_RESCHEDULER_MAX_TIME,
     PROXY_BUDGET_RESCHEDULER_MIN_TIME,
+    PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
 )
 from litellm.exceptions import RejectedRequestError
 from litellm.integrations.custom_guardrail import ModifyResponseException
@@ -319,7 +320,10 @@ from litellm.proxy.common_utils.openai_endpoint_utils import (
 from litellm.proxy.common_utils.proxy_state import ProxyState
 from litellm.proxy.common_utils.reset_budget_job import ResetBudgetJob
 from litellm.proxy.common_utils.swagger_utils import ERROR_RESPONSES
-from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
+from litellm.proxy.common_utils.timezone_utils import (
+    get_budget_reset_settings,
+    get_budget_reset_time,
+)
 from litellm.proxy.common_utils.user_api_key_cache import (
     UserApiKeyCache,
     get_management_object_ttl,
@@ -1998,6 +2002,7 @@ proxy_budget_rescheduler_min_time = PROXY_BUDGET_RESCHEDULER_MIN_TIME
 proxy_budget_rescheduler_max_time = PROXY_BUDGET_RESCHEDULER_MAX_TIME
 proxy_batch_polling_interval = PROXY_BATCH_POLLING_INTERVAL
 proxy_batch_write_at = PROXY_BATCH_WRITE_AT
+proxy_config_reload_interval_seconds = PROXY_CONFIG_RELOAD_INTERVAL_SECONDS
 litellm_master_key_hash = None
 disable_spend_logs = False
 jwt_handler = JWTHandler()
@@ -3879,7 +3884,7 @@ class ProxyConfig:
         del config["include"]
         return config
 
-    async def save_config(self, new_config: dict):
+    async def save_config(self, new_config: dict, include_env_vars: bool = False):
         global prisma_client, general_settings, user_config_file_path, store_model_in_db
         # Load existing config
         ## DB - writes valid config to db
@@ -3896,6 +3901,17 @@ class ProxyConfig:
             # Make a copy to avoid mutating the original config
             config_to_save = new_config.copy()
 
+            # environment_variables are persisted to the DB only when a caller
+            # explicitly opts in. Most callers reach save_config after
+            # get_config() merged YAML + OS env into new_config (with
+            # os.environ/ placeholders already resolved to plaintext), so
+            # persisting them here would snapshot file/container env vars into
+            # a config row that then shadows those sources on every restart.
+            # The dedicated /config/update path writes env vars directly, so
+            # no current caller needs include_env_vars=True.
+            if not include_env_vars:
+                config_to_save.pop("environment_variables", None)
+
             # SECURITY: Always encrypt environment_variables before DB write.
             # _encrypt_env_variables_for_db is idempotent — a caller that
             # already encrypted the values (or re-submitted ciphertext read
@@ -3913,6 +3929,38 @@ class ProxyConfig:
             with open(f"{user_config_file_path}", "w") as config_file:
                 yaml.dump(new_config, config_file, default_flow_style=False)
 
+    async def save_environment_variables(self, updates: dict[str, str | None]) -> None:
+        """Persist specific environment variables to the DB config row.
+
+        Each key in ``updates`` is written to the ``environment_variables``
+        config row; a ``None`` value deletes that key. Env vars the caller does
+        not name are preserved, so a caller that owns a couple of keys can
+        update just those without snapshotting unrelated (YAML/OS-sourced)
+        values the way a full ``save_config`` write would. No-op when config is
+        not DB-backed.
+        """
+        global prisma_client, general_settings, store_model_in_db
+        if prisma_client is None or not (general_settings.get("store_model_in_db", False) is True or store_model_in_db):
+            return
+
+        row = await ConfigRepository(prisma_client).table.find_first(where={"param_name": "environment_variables"})
+        existing: dict = dict(row.param_value) if row is not None and row.param_value is not None else {}
+
+        to_set = {k: v for k, v in updates.items() if v is not None}
+        encrypted = self._encrypt_env_variables_for_db(environment_variables=to_set) if to_set else {}
+        deleted_keys = {k for k, v in updates.items() if v is None}
+        merged = {**{k: v for k, v in existing.items() if k not in deleted_keys}, **encrypted}
+
+        serialized = json.dumps(merged)
+        await ConfigRepository(prisma_client).table.upsert(
+            where={"param_name": "environment_variables"},
+            data={
+                "create": {"param_name": "environment_variables", "param_value": serialized},
+                "update": {"param_value": serialized},
+            },
+        )
+        await invalidate_config_param("environment_variables")
+
     def _check_for_os_environ_vars(
         self, config: dict, depth: int = 0, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH
     ) -> dict:
@@ -4291,6 +4339,7 @@ class ProxyConfig:
             open_telemetry_logger, \
             health_check_details, \
             proxy_batch_polling_interval, \
+            proxy_config_reload_interval_seconds, \
             config_passthrough_endpoints
 
         config: dict = await self.get_config(config_file_path=config_file_path)
@@ -4597,6 +4646,13 @@ class ProxyConfig:
                     litellm.json_logs = True
                     litellm._turn_on_json()
                     verbose_proxy_logger.debug(f"{blue_color_code} Enabled JSON logging via config{reset_color_code}")
+                elif key == "budget_reset_time":
+                    from litellm.proxy.common_utils.timezone_utils import (
+                        parse_budget_reset_time,
+                    )
+
+                    parse_budget_reset_time(value)
+                    setattr(litellm, key, value)
                 else:
                     verbose_proxy_logger.debug(
                         f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, value, is_full_admin=False)}{reset_color_code}"
@@ -4773,6 +4829,10 @@ class ProxyConfig:
             )
             ## BATCH WRITER ##
             proxy_batch_write_at = general_settings.get("proxy_batch_write_at", proxy_batch_write_at)
+            ## DB CONFIG RELOAD INTERVAL ##
+            proxy_config_reload_interval_seconds = general_settings.get(
+                "proxy_config_reload_interval_seconds", proxy_config_reload_interval_seconds
+            )
             ## DISABLE SPEND LOGS ## - gives a perf improvement
             disable_spend_logs = general_settings.get("disable_spend_logs", disable_spend_logs)
             ### BACKGROUND HEALTH CHECKS ###
@@ -7868,6 +7928,7 @@ class ProxyStartupEvent:
             budget_reset_job = ResetBudgetJob(
                 proxy_logging_obj=proxy_logging_obj,
                 prisma_client=prisma_client,
+                reset_settings=get_budget_reset_settings(),
             )
 
             scheduler.add_job(
@@ -7944,12 +8005,20 @@ class ProxyStartupEvent:
                 verbose_proxy_logger.debug("Failed to check DB for store_model_in_db: %s", str(e))
 
         if store_model_in_db is True:
+            config_reload_interval_seconds = proxy_config_reload_interval_seconds
+            if not isinstance(config_reload_interval_seconds, int) or config_reload_interval_seconds <= 0:
+                verbose_proxy_logger.warning(
+                    "proxy_config_reload_interval_seconds=%s must be a positive integer; falling back to 30s",
+                    config_reload_interval_seconds,
+                )
+                config_reload_interval_seconds = 30
+
             # MEMORY LEAK FIX: Increase interval from 10s to 30s minimum
             # Frequent polling was causing excessive memory allocations
             scheduler.add_job(
                 proxy_config.add_deployment,
                 "interval",
-                seconds=30,  # increased from 10s to reduce memory pressure
+                seconds=config_reload_interval_seconds,
                 # REMOVED jitter parameter - major cause of memory leak
                 args=[prisma_client, proxy_logging_obj],
                 id="add_deployment_job",
@@ -7964,7 +8033,7 @@ class ProxyStartupEvent:
             scheduler.add_job(
                 proxy_config.get_credentials,
                 "interval",
-                seconds=30,  # increased from 10s to reduce memory pressure
+                seconds=config_reload_interval_seconds,
                 # REMOVED jitter parameter - major cause of memory leak
                 args=[prisma_client],
                 id="get_credentials_job",
@@ -14998,6 +15067,7 @@ async def get_config_list(
         "global_max_parallel_requests": {"type": "Integer"},
         "max_request_size_mb": {"type": "Integer"},
         "max_response_size_mb": {"type": "Integer"},
+        "proxy_config_reload_interval_seconds": {"type": "Integer"},
         "pass_through_endpoints": {"type": "PydanticModel"},
         "store_model_in_db": {"type": "Boolean"},
         "store_prompts_in_spend_logs": {"type": "Boolean"},

litellm/proxy/schema.prisma

@@ -403,6 +403,15 @@ model LiteLLM_MCPServerOAuthClient {
   updated_at  DateTime @default(now()) @updatedAt @map("updated_at")
 }
 
+// The enterprise IdP identity assertion captured at SSO login, one row per user.
+// assertion_b64 is an encrypted JSON payload: {id_token, refresh_token?, issuer?, expires_at?}.
+model LiteLLM_SSOIdentityAssertion {
+  user_id       String   @id
+  assertion_b64 String
+  created_at    DateTime @default(now()) @map("created_at")
+  updated_at    DateTime @default(now()) @updatedAt @map("updated_at")
+}
+
 // Generate Tokens for Proxy
 model LiteLLM_VerificationToken {
     token      String   @id

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -1,6 +1,8 @@
 #### CRUD ENDPOINTS for UI Settings #####
 import asyncio
 import json
+import os
+from collections.abc import Mapping
 from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
 from urllib.parse import urlparse
 
@@ -35,6 +37,44 @@ _SSO_SENSITIVE_FIELDS: Set[str] = {
     "generic_client_secret",
 }
 
+# Maps each UIThemeConfig field to the env var the UI branding path reads it
+# from. /update/ui_theme_settings writes both the stored ui_theme_config and
+# these env vars, so /get/ui_theme_settings resolves the same env vars to
+# reflect a deployment branded purely through process env.
+_UI_THEME_FIELD_ENV_VARS: dict[str, str] = {
+    "logo_url": "UI_LOGO_PATH",
+    "favicon_url": "LITELLM_FAVICON_URL",
+}
+
+
+def _is_public_http_url(value: str | None) -> bool:
+    """Whether a value is a plain http(s) URL with a host, safe to disclose publicly."""
+    if not isinstance(value, str) or not value.strip():
+        return False
+    parsed = urlparse(value.strip())
+    return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+
+
+def _resolve_ui_theme_field(stored_values: Mapping[str, Any], field_name: str) -> str | None:
+    """Resolve one UI theme field to the value the branding path actually uses.
+
+    The stored ui_theme_config wins; a field absent or blank there falls back to
+    the process environment. The branding path reads the env var, and stored
+    settings reach it by being pushed into the environment on save, so a value
+    supplied only as a process env var is live even though no stored entry exists.
+
+    This endpoint is unauthenticated, so the env fallback only surfaces a public
+    http(s) URL: an operator can point UI_LOGO_PATH at a local filesystem path
+    (the branding path serves it server-side), and that path must not be
+    disclosed to anonymous callers. A stored value is already validated as a
+    public URL on write, so it passes through.
+    """
+    stored = stored_values.get(field_name)
+    if isinstance(stored, str) and stored.strip():
+        return stored
+    env_value = os.environ.get(_UI_THEME_FIELD_ENV_VARS[field_name])
+    return env_value if _is_public_http_url(env_value) else None
+
 
 class IPAddress(BaseModel):
     ip: str
@@ -977,12 +1017,19 @@ async def get_ui_theme_settings():
     # Load existing config
     config = await proxy_config.get_config()
 
-    return await _get_settings_with_schema(
+    result = await _get_settings_with_schema(
         settings_key="ui_theme_config",
         settings_class=UIThemeConfig,
         config=config,
     )
 
+    stored_values = result.get("values", {})
+    result["values"] = {
+        **stored_values,
+        **{field: _resolve_ui_theme_field(stored_values, field) for field in _UI_THEME_FIELD_ENV_VARS},
+    }
+    return result
+
 
 def _validate_public_image_url(value: Optional[str], field_name: str) -> None:
     """
@@ -1041,13 +1088,6 @@ async def update_ui_theme_settings(
     config = await proxy_config.get_config()
     before_theme = config.get("litellm_settings", {}).get("ui_theme_config")
 
-    # Update config with UI theme settings
-    if "general_settings" not in config:
-        config["general_settings"] = {}
-
-    if "environment_variables" not in config:
-        config["environment_variables"] = {}
-
     # Convert theme config to dict
     theme_data = theme_config.model_dump(exclude_none=True)
 
@@ -1056,55 +1096,29 @@ async def update_ui_theme_settings(
         config["litellm_settings"] = {}
     config["litellm_settings"]["ui_theme_config"] = theme_data
 
-    # Update UI_LOGO_PATH environment variable if logo_url is provided
-    # If logo_url is empty string, None, or null, remove the environment variable to use default
-    logo_url = theme_data.get("logo_url")
-    verbose_proxy_logger.debug(f"Updating logo_url: {logo_url}")
+    # UI_LOGO_PATH and LITELLM_FAVICON_URL are the only environment variables
+    # this endpoint owns. A non-empty value sets the var; an empty or missing
+    # one clears it back to the default. Apply to the live process immediately,
+    # then persist only these two keys so an unrelated env var (a YAML/OS value
+    # merged in by get_config) is never snapshotted into the DB.
+    def _clean(url: str | None) -> str | None:
+        return url if url is not None and url.strip() else None
 
-    if (
-        logo_url and isinstance(logo_url, str) and logo_url.strip()
-    ):  # Check if logo_url exists and is not empty/whitespace
-        config["environment_variables"]["UI_LOGO_PATH"] = logo_url
-        os.environ["UI_LOGO_PATH"] = logo_url
-        verbose_proxy_logger.debug(f"Set UI_LOGO_PATH to: {logo_url}")
-    else:
-        # Remove the environment variable to restore default logo
-        if "UI_LOGO_PATH" in config.get("environment_variables", {}):
-            del config["environment_variables"]["UI_LOGO_PATH"]
-            verbose_proxy_logger.debug("Removed UI_LOGO_PATH from config")
-        if "UI_LOGO_PATH" in os.environ:
-            del os.environ["UI_LOGO_PATH"]
-            verbose_proxy_logger.debug("Removed UI_LOGO_PATH from environment")
+    env_updates: dict[str, str | None] = {
+        "UI_LOGO_PATH": _clean(theme_config.logo_url),
+        "LITELLM_FAVICON_URL": _clean(theme_config.favicon_url),
+    }
+    for env_key, env_value in env_updates.items():
+        if env_value is not None:
+            os.environ[env_key] = env_value
+        else:
+            os.environ.pop(env_key, None)
 
-    # Update LITELLM_FAVICON_URL environment variable if favicon_url is provided
-    favicon_url = theme_data.get("favicon_url")
-    verbose_proxy_logger.debug(f"Updating favicon_url: {favicon_url}")
-
-    if (
-        favicon_url and isinstance(favicon_url, str) and favicon_url.strip()
-    ):  # Check if favicon_url exists and is not empty/whitespace
-        config["environment_variables"]["LITELLM_FAVICON_URL"] = favicon_url
-        os.environ["LITELLM_FAVICON_URL"] = favicon_url
-        verbose_proxy_logger.debug(f"Set LITELLM_FAVICON_URL to: {favicon_url}")
-    else:
-        # Remove the environment variable to restore default favicon
-        if "LITELLM_FAVICON_URL" in config.get("environment_variables", {}):
-            del config["environment_variables"]["LITELLM_FAVICON_URL"]
-            verbose_proxy_logger.debug("Removed LITELLM_FAVICON_URL from config")
-        if "LITELLM_FAVICON_URL" in os.environ:
-            del os.environ["LITELLM_FAVICON_URL"]
-            verbose_proxy_logger.debug("Removed LITELLM_FAVICON_URL from environment")
-
-    # Handle environment variable encryption if needed
-    stored_config = config.copy()
-    if "environment_variables" in stored_config and len(stored_config["environment_variables"]) > 0:
-        # Only encrypt if there are environment variables to encrypt
-        stored_config["environment_variables"] = proxy_config._encrypt_env_variables(
-            environment_variables=stored_config["environment_variables"]
-        )
-
-    # Save the updated config
-    await proxy_config.save_config(new_config=stored_config)
+    # Persist the theme config (litellm_settings). save_config defaults to
+    # include_env_vars=False, so it does not snapshot environment_variables.
+    await proxy_config.save_config(new_config=config)
+    # Persist only the two owned env vars, merged against the existing DB row.
+    await proxy_config.save_environment_variables(env_updates)
 
     asyncio.create_task(
         create_config_audit_log(

litellm/types/guardrails.py

@@ -826,6 +826,13 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
             "while fail_on_error still governs real Model Armor API errors. Default False blocks them."
         ),
     )
+    sanitize_error_detail: Optional[bool] = Field(
+        default=True,
+        description=(
+            "For guardrail='model_armor': omit the raw Model Armor response from "
+            "caller-facing errors and logs by default. Set False to restore verbose output."
+        ),
+    )
 
     additional_provider_specific_params: Optional[Dict[str, Any]] = Field(
         default=None,

litellm/types/interactions/generated.py

@@ -173,6 +173,7 @@ class Status1(Enum):
     cancelled = "cancelled"
     incomplete = "incomplete"
     budget_exceeded = "budget_exceeded"
+    queued = "queued"
 
 
 class InteractionStatusUpdate(BaseModel):
@@ -341,6 +342,7 @@ class Status3(Enum):
     CANCELLED = "cancelled"
     INCOMPLETE = "incomplete"
     BUDGET_EXCEEDED = "budget_exceeded"
+    QUEUED = "queued"
 
 
 class ModelOption(RootModel[str]):

litellm/types/proxy/guardrails/guardrail_hooks/model_armor.py

@@ -20,6 +20,13 @@ class ModelArmorGuardrailConfigModel(GuardrailConfigModel):
         default=True,
         description="Whether to fail the request if Model Armor encounters an error",
     )
+    sanitize_error_detail: Optional[bool] = Field(
+        default=True,
+        description=(
+            "Omit the raw Model Armor response from caller-facing errors and logs "
+            "by default. Set False to restore verbose output."
+        ),
+    )
 
     @staticmethod
     def ui_friendly_name() -> str:

litellm/utils.py

@@ -1864,7 +1864,9 @@ def client(original_function):
                     except Exception:
                         pass
 
-            setattr(e, "num_retries", num_retries)  ## IMPORTANT: returns the deployment's num_retries to the router
+            deployment_num_retries = kwargs.get("num_retries")
+            if deployment_num_retries is not None:
+                setattr(e, "num_retries", deployment_num_retries)
 
             timeout = _get_wrapper_timeout(kwargs=kwargs, exception=e)
             setattr(e, "timeout", timeout)

model_prices_and_context_window.json

@@ -17644,6 +17644,61 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "cache_read_input_token_cost_flex": 2e-08,
+        "cache_read_input_token_cost_priority": 5e-08,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_batches": 1.5e-07,
+        "input_cost_per_token_flex": 1.5e-07,
+        "input_cost_per_token_priority": 5.4e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_batches": 1.25e-06,
+        "output_cost_per_token_flex": 1.25e-06,
+        "output_cost_per_token_priority": 4.5e-06,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,
@@ -18311,6 +18366,60 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "vertex_ai/gemini-3.6-flash": {
+        "cache_read_input_token_cost": 1.5e-07,
+        "cache_read_input_token_cost_flex": 7.5e-08,
+        "input_cost_per_token": 1.5e-06,
+        "input_cost_per_token_batches": 7.5e-07,
+        "input_cost_per_token_flex": 7.5e-07,
+        "litellm_provider": "vertex_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 7.5e-06,
+        "output_cost_per_token": 7.5e-06,
+        "output_cost_per_token_batches": 3.75e-06,
+        "output_cost_per_token_flex": 3.75e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "input_cost_per_token_priority": 2.7e-06,
+        "output_cost_per_token_priority": 1.35e-05,
+        "cache_read_input_token_cost_priority": 2.7e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "vertex_ai/gemini-3.1-pro-preview": {
         "cache_read_input_token_cost": 2e-07,
         "cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -19663,6 +19772,63 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini/gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "cache_read_input_token_cost_flex": 2e-08,
+        "cache_read_input_token_cost_priority": 5e-08,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_batches": 1.5e-07,
+        "input_cost_per_token_flex": 1.5e-07,
+        "input_cost_per_token_priority": 5.4e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_batches": 1.25e-06,
+        "output_cost_per_token_flex": 1.25e-06,
+        "output_cost_per_token_priority": 4.5e-06,
+        "rpm": 15,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "tpm": 250000,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "gemini/gemini-3-flash-preview": {
         "cache_read_input_token_cost": 5e-08,
         "input_cost_per_audio_token": 1e-06,
@@ -19769,6 +19935,63 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini/gemini-3.6-flash": {
+        "cache_read_input_token_cost": 1.5e-07,
+        "cache_read_input_token_cost_flex": 7.5e-08,
+        "input_cost_per_token": 1.5e-06,
+        "input_cost_per_token_batches": 7.5e-07,
+        "input_cost_per_token_flex": 7.5e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 7.5e-06,
+        "output_cost_per_token": 7.5e-06,
+        "output_cost_per_token_batches": 3.75e-06,
+        "output_cost_per_token_flex": 3.75e-06,
+        "rpm": 2000,
+        "source": "https://ai.google.dev/pricing/gemini-3",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_output": false,
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "tpm": 800000,
+        "input_cost_per_token_priority": 2.7e-06,
+        "output_cost_per_token_priority": 1.35e-05,
+        "cache_read_input_token_cost_priority": 2.7e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "gemini/gemini-omni-flash-preview": {
         "input_cost_per_audio_token": 1.5e-06,
         "input_cost_per_token": 1.5e-06,
@@ -20049,6 +20272,61 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini-3.6-flash": {
+        "cache_read_input_token_cost": 1.5e-07,
+        "cache_read_input_token_cost_flex": 7.5e-08,
+        "input_cost_per_token": 1.5e-06,
+        "input_cost_per_token_batches": 7.5e-07,
+        "input_cost_per_token_flex": 7.5e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 7.5e-06,
+        "output_cost_per_token": 7.5e-06,
+        "output_cost_per_token_batches": 3.75e-06,
+        "output_cost_per_token_flex": 3.75e-06,
+        "source": "https://ai.google.dev/pricing/gemini-3",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_output": false,
+        "supports_audio_input": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "input_cost_per_token_priority": 2.7e-06,
+        "output_cost_per_token_priority": 1.35e-05,
+        "cache_read_input_token_cost_priority": 2.7e-07,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "gemini/gemini-2.5-pro-preview-tts": {
         "cache_read_input_token_cost": 1.25e-07,
         "cache_read_input_token_cost_above_200k_tokens": 2.5e-07,
@@ -37332,6 +37610,61 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "vertex_ai/gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "cache_read_input_token_cost_flex": 2e-08,
+        "cache_read_input_token_cost_priority": 5e-08,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_batches": 1.5e-07,
+        "input_cost_per_token_flex": 1.5e-07,
+        "input_cost_per_token_priority": 5.4e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_batches": 1.25e-06,
+        "output_cost_per_token_flex": 1.25e-06,
+        "output_cost_per_token_priority": 4.5e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": false,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_url_context": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_native_streaming": true,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.014,
+            "search_context_size_medium": 0.014,
+            "search_context_size_high": 0.014
+        },
+        "web_search_billing_unit": "per_query"
+    },
     "vertex_ai/deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.94.0"
+version = "1.95.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -62,7 +62,7 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.79",
+    "litellm-proxy-extras==0.4.80",
     "litellm-enterprise==0.1.51",
     "RestrictedPython>=8.1,<9.0",
     "rich>=13.9.4,<14.0",
@@ -117,6 +117,14 @@ stt-nvidia-riva = [
     "numpy>=1.26.0",
 ]
 google = ["google-cloud-aiplatform>=1.133.0,<2.0"]
+bedrock-realtime = [
+    # Bedrock Nova Sonic realtime (speech-to-speech) uses the
+    # InvokeModelWithBidirectionalStream API, which boto3 cannot do. This
+    # experimental AWS SDK (with its smithy-* deps, pulled transitively)
+    # provides the bidirectional stream; imported lazily in the realtime
+    # handler so litellm core stays usable without it.
+    "aws-sdk-bedrock-runtime>=0.7.0,<0.8.0; python_version >= '3.12'",
+]
 proxy-runtime = [
     # Historically bundled in the proxy Docker images via requirements.txt.
     # Keep these in a dedicated extra so uv-based images preserve the same
@@ -289,7 +297,7 @@ members = ["enterprise", "litellm-proxy-extras"]
 profile = "black"
 
 [tool.commitizen]
-version = "1.94.0"
+version = "1.95.0"
 version_files = [
     "pyproject.toml:^version",
 ]

ruff-strict-budget.json

@@ -93,7 +93,7 @@
     "limit": 33
   },
   "DTZ005": {
-    "limit": 244
+    "limit": 241
   },
   "DTZ006": {
     "limit": 13

schema.prisma

@@ -403,6 +403,15 @@ model LiteLLM_MCPServerOAuthClient {
   updated_at  DateTime @default(now()) @updatedAt @map("updated_at")
 }
 
+// The enterprise IdP identity assertion captured at SSO login, one row per user.
+// assertion_b64 is an encrypted JSON payload: {id_token, refresh_token?, issuer?, expires_at?}.
+model LiteLLM_SSOIdentityAssertion {
+  user_id       String   @id
+  assertion_b64 String
+  created_at    DateTime @default(now()) @map("created_at")
+  updated_at    DateTime @default(now()) @updatedAt @map("updated_at")
+}
+
 // Generate Tokens for Proxy
 model LiteLLM_VerificationToken {
     token      String   @id

scripts/pre_commit_lint.sh

@@ -5,6 +5,7 @@
 # gating CI checks, so a clean run means a green CI lint:
 #   - litellm/ Python staged -> `make lint` (test-linting.yml's lint job)
 #   - tests/e2e Python staged -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step)
+#                                + raw HTTP client ban (test-code-quality.yml's check_e2e_no_raw_requests)
 #   - dashboard staged        -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint)
 #   - proxy/types staged      -> regenerate dashboard API types and fail on drift (check-ui-api-types.yml)
 #
@@ -112,6 +113,12 @@ if [ -n "$e2e_py_files" ] && [ -z "$litellm_py_files" ]; then
     make lint-e2e-basedpyright || { echo "✗ tests/e2e basedpyright failed. Fix the errors above, then re-run make pre-commit." >&2; status=1; }
 fi
 
+if [ -n "$e2e_py_files" ]; then
+    echo "pre-commit: checking tests/e2e raw HTTP client ban (check_e2e_no_raw_requests)"
+    uv run --no-sync python tests/code_coverage_tests/check_e2e_no_raw_requests.py \
+        || { echo "✗ Raw HTTP client import in tests/e2e. Route the call through tests/e2e/e2e_http.py, then re-run make pre-commit." >&2; status=1; }
+fi
+
 if [ -n "$ui_prettier_files" ] || [ -n "$ui_eslint_files" ]; then
     echo "pre-commit: linting dashboard (prettier + eslint + lint budgets)"
     if [ ! -d ui/litellm-dashboard/node_modules ]; then

tests/code_coverage_tests/check_e2e_no_raw_requests.py

@@ -0,0 +1,81 @@
+"""tests/e2e routes every HTTP call through the typed transport (e2e_http.py), so
+raw HTTP client imports (requests, urllib.request, httpx, aiohttp, http.client) are
+banned in suite code. Importing requests' exception types for catching is fine
+anywhere; a small allowlist grandfathers the files that legitimately make raw calls
+(the transport itself, the root conftest liveness probe, and the claude_code version
+resolver's constant registry URL fetch). Referenced by tests/e2e/CLAUDE.md."""
+
+from __future__ import annotations
+
+import ast
+import sys
+from pathlib import Path
+
+E2E_DIR = Path(__file__).resolve().parents[1] / "e2e"
+
+BANNED_MODULES = ("requests", "urllib.request", "http.client", "httpx", "aiohttp")
+
+ALLOWED_RAW_CLIENT_FILES = {
+    "e2e_http.py": ("requests",),
+    "conftest.py": ("requests",),
+    "claude_code/pr_gate_version_resolver.py": ("urllib.request",),
+}
+
+EXCEPTION_ONLY_NAMES = frozenset({"RequestException", "ConnectionError", "Timeout", "HTTPError"})
+
+
+def _is_banned(module: str) -> bool:
+    return any(module == banned or module.startswith(banned + ".") for banned in BANNED_MODULES)
+
+
+def _banned_imports(tree: ast.Module) -> tuple[tuple[str, int], ...]:
+    plain = tuple(
+        (alias.name, node.lineno)
+        for node in ast.walk(tree)
+        if isinstance(node, ast.Import)
+        for alias in node.names
+        if _is_banned(alias.name)
+    )
+    from_imports = tuple(
+        (node.module, node.lineno)
+        for node in ast.walk(tree)
+        if isinstance(node, ast.ImportFrom)
+        and node.module is not None
+        and _is_banned(node.module)
+        and not all(alias.name in EXCEPTION_ONLY_NAMES for alias in node.names)
+    )
+    return plain + from_imports
+
+
+def _violations_in(path: Path) -> tuple[str, ...]:
+    relative = path.relative_to(E2E_DIR).as_posix()
+    allowed = ALLOWED_RAW_CLIENT_FILES.get(relative, ())
+    tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+    return tuple(
+        f"tests/e2e/{relative}:{lineno}: raw HTTP client import '{module}'"
+        for module, lineno in _banned_imports(tree)
+        if module not in allowed
+    )
+
+
+def main() -> int:
+    violations = tuple(
+        violation
+        for path in sorted(E2E_DIR.rglob("*.py"))
+        for violation in _violations_in(path)
+    )
+    for violation in violations:
+        print(violation)
+    if violations:
+        print(
+            f"\n{len(violations)} raw HTTP client import(s) in tests/e2e. "
+            "Route the call through tests/e2e/e2e_http.py (get_external for absolute "
+            "third-party URLs) so it gets the typed Result handling."
+        )
+        return 1
+    print("tests/e2e raw HTTP client check passed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/code_coverage_tests/recursive_detector.py

@@ -55,6 +55,8 @@ IGNORE_FUNCTIONS = [
     "_freeze_for_dedupe",  # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap.
     "apply_json_merge_patch",  # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap.
     "_filter_mcp_argument_value",  # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the MCP call at the cap.
+    "_redact_scanned_content",  # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap.
+    "_iter_fallback_targets",  # max depth set (2 * ROUTER_MAX_FALLBACKS); fails closed by raising ValueError at the cap.
 ]
 
 

tests/e2e/CLAUDE.md

@@ -13,11 +13,13 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
 - `realtime/` - realtime websocket sessions, including the pipecat audio path
 - `quota_management/` - quota enforcement and accounting, one subfolder per behavior: `ratelimit/` (rpm/tpm blocks, window reset, pacing headers on live traffic), `budgets/` (budget definition, enforcement, and reset windows: key, team, tag, soft, multi-window), and `spend_tracking/` (spend logging and cost attribution on `/spend/*`)
 - `management/` - key/team/user/organization management routes: create/update/delete persistence via the info routes, team membership, and llm-only-key route denials (API surface; not Playwright)
+- `a2a/` - the A2A (agent-to-agent) surface: admin registration via `/v1/agents`, proxy-fronted card discovery at `/.well-known/agent-card.json`, and JSON-RPC `message/send` invocation, driving agents backed by the litellm completion bridge (a real provider) and asserting protocol-version normalization (0.3 vs 1.0)
 - `mcp/` - the MCP server surface over api_key auth against the real Datadog remote MCP server only (see "MCP suite: real Datadog only" below)
 - `logging/` - logging-integration delivery (datadog and friends)
 - `security/` - secret handling and log-leak protection
 - `router/` - routing and reliability behavior (fallbacks, cooldowns)
-- `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO; marked `load` so the parent conftest collects it last and it never perturbs latency-sensitive suites
+- `load/` - throughput/performance under concurrency: drives real concurrent traffic through the whole stack with Locust and asserts a throughput SLO; marked `load` so the parent conftest collects it last and it never perturbs latency-sensitive suites. Also home of the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`): Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; additionally marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, because it spends real provider money (driven by `.github/workflows/weekly_load_anomaly.yml`)
+- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
 - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
 - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
 
@@ -130,7 +132,7 @@ reliability.<behavior>.<variant>.<assertion>
   behavior  : fallback | retry | cooldown | timeout | routing | cache | circuit_breaker | perf
   variant   : <trigger>   5xx | context_window | content_policy | 429 | timeout
               <strategy>  simple_shuffle | usage_based | latency_based | cost_based | least_busy
-              <dimension> latency | throughput   (perf only; SLO/threshold assertion, not binary)
+              <dimension> latency | throughput | session_anomaly   (perf only; SLO/threshold assertion, not binary)
   assertion : routes_to_fallback | succeeds_within_retries | picks_under_tpm | returns_cached
               | trips_then_recovers | under_slo
   e.g.  reliability.fallback.context_window.routes_to_fallback     exercised_on=[chat_completions]

tests/e2e/a2a/a2a_client.py

@@ -0,0 +1,292 @@
+"""Client for the proxy's A2A (agent-to-agent) surface.
+
+An A2A agent is registered admin-side via POST /v1/agents with an agent card and
+litellm_params; the proxy fronts it at /a2a/{id}, serving a proxy-owned agent card
+at /.well-known/agent-card.json and accepting A2A JSON-RPC calls at /a2a/{id}. This
+suite registers agents backed by the litellm_completion_bridge (custom_llm_provider
++ model), so message/send runs a real provider completion and comes back in the
+agent's pinned A2A protocol version. The A2A request/response models are co-located
+here because only this suite uses them.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from e2e_http import NoBody, Result, get_external, is_ok
+from proxy_client import ProxyClient
+
+
+class A2ACapabilities(BaseModel):
+    streaming: bool | None = None
+    push_notifications: bool | None = Field(default=None, serialization_alias="pushNotifications")
+
+
+class A2ASkill(BaseModel):
+    id: str
+    name: str
+    description: str
+    tags: list[str]
+    examples: list[str] | None = None
+
+
+class A2AProvider(BaseModel):
+    organization: str
+    url: str
+
+
+class AgentCardParams(BaseModel):
+    """The upstream agent card an admin registers. `protocolVersion` is the field the
+    proxy validates against SUPPORTED_A2A_PROTOCOL_VERSIONS on registration."""
+
+    protocol_version: str = Field(serialization_alias="protocolVersion")
+    name: str
+    description: str
+    version: str
+    url: str | None = None
+    capabilities: A2ACapabilities = A2ACapabilities()
+    skills: list[A2ASkill]
+    default_input_modes: list[str] = Field(default=["text"], serialization_alias="defaultInputModes")
+    default_output_modes: list[str] = Field(default=["text"], serialization_alias="defaultOutputModes")
+    preferred_transport: str | None = Field(default=None, serialization_alias="preferredTransport")
+
+
+class UpstreamAgentCard(BaseModel):
+    """A real published agent card parsed from a public /.well-known endpoint. Keys on
+    the A2A wire aliases so `model_validate_json` reads the served JSON and
+    `model_dump(by_alias=True)` re-emits it unchanged for verbatim registration; it is
+    only ever fetched-and-validated, never hand-constructed, so aliasing on the wire
+    names does not affect any call site."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    protocol_version: str = Field(alias="protocolVersion")
+    name: str
+    description: str
+    version: str
+    url: str
+    provider: A2AProvider | None = None
+    documentation_url: str | None = Field(default=None, alias="documentationUrl")
+    capabilities: A2ACapabilities = A2ACapabilities()
+    skills: list[A2ASkill]
+    default_input_modes: list[str] = Field(default=["text"], alias="defaultInputModes")
+    default_output_modes: list[str] = Field(default=["text"], alias="defaultOutputModes")
+    preferred_transport: str | None = Field(default=None, alias="preferredTransport")
+
+
+class A2ABridgeParams(BaseModel):
+    """litellm_params that route the agent through the completion bridge: an A2A
+    message/send is transformed into a litellm.acompletion against this provider."""
+
+    model_config = ConfigDict(protected_namespaces=())
+
+    custom_llm_provider: str
+    model: str
+
+
+class AgentRegisterBody(BaseModel):
+    agent_name: str
+    agent_card_params: AgentCardParams | UpstreamAgentCard
+    litellm_params: A2ABridgeParams | None = None
+
+
+class A2ASecurityScheme(BaseModel):
+    type: str
+    scheme: str
+
+
+class A2AInterface(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    url: str
+    protocol_version: str | None = Field(default=None, alias="protocolVersion")
+
+
+class ServedAgentCard(BaseModel):
+    """The proxy-owned card, either nested under a registration response's
+    `agent_card_params` or served raw at /.well-known/agent-card.json. The proxy
+    rewrites `url`/`supportedInterfaces` to itself and replaces the security scheme
+    with its own virtual-key bearer scheme."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    protocol_version: str = Field(alias="protocolVersion")
+    name: str
+    url: str | None = None
+    security_schemes: dict[str, A2ASecurityScheme] | None = Field(default=None, alias="securitySchemes")
+    security: list[dict[str, list[str]]] | None = None
+    supported_interfaces: list[A2AInterface] | None = Field(default=None, alias="supportedInterfaces")
+
+
+class AgentResponse(BaseModel):
+    agent_id: str
+    agent_name: str
+    agent_card_params: ServedAgentCard
+
+
+class A2ATextPart(BaseModel):
+    kind: str = "text"
+    text: str
+
+
+class A2ASearchPropertiesParams(BaseModel):
+    """The strict param schema of the published property agent's `search_properties`
+    skill (unknown keys are rejected upstream), so a natural-language query like
+    "properties for sale in SF under $2M" is expressed as typed fields."""
+
+    un_locode: str | None = None
+    service_type: str | None = None
+    property_type: str | None = None
+    bedrooms_min: int | None = None
+    asking_price_max: float | None = None
+    limit: int | None = None
+
+
+class A2ASkillInvocation(BaseModel):
+    skill: str
+    params: A2ASearchPropertiesParams
+
+
+class A2ADataPart(BaseModel):
+    kind: str = "data"
+    data: A2ASkillInvocation
+
+
+class A2AOutboundMessage(BaseModel):
+    role: str = "user"
+    parts: list[A2ATextPart | A2ADataPart]
+    message_id: str = Field(serialization_alias="messageId")
+
+
+class A2AMessageSendParams(BaseModel):
+    message: A2AOutboundMessage
+
+
+class A2AJsonRpcRequest(BaseModel):
+    jsonrpc: str = "2.0"
+    id: str
+    method: str = "message/send"
+    params: A2AMessageSendParams
+
+
+class A2AResponsePart(BaseModel):
+    kind: str | None = None
+    text: str | None = None
+
+
+class A2AResponseMessage(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    message_id: str | None = Field(default=None, alias="messageId")
+    role: str | None = None
+    parts: list[A2AResponsePart] = []
+
+
+class A2ATaskStatus(BaseModel):
+    state: str | None = None
+    message: A2AResponseMessage | None = None
+
+
+class A2AResult(BaseModel):
+    """A message/send result. In 0.3 the message fields sit directly on the result
+    (`kind`/`role`/`parts`); in 1.0 they are nested under `message`; a real agent that
+    runs a task replies with a `task` whose agent text lives on `status.message`.
+    `text` reads the agent's reply from whichever shape the served version produced."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    kind: str | None = None
+    role: str | None = None
+    message_id: str | None = Field(default=None, alias="messageId")
+    parts: list[A2AResponsePart] = []
+    message: A2AResponseMessage | None = None
+    status: A2ATaskStatus | None = None
+
+    @property
+    def text(self) -> str:
+        if self.message is not None:
+            parts = self.message.parts
+        elif self.parts:
+            parts = self.parts
+        elif self.status is not None and self.status.message is not None:
+            parts = self.status.message.parts
+        else:
+            parts = []
+        return "".join(part.text or "" for part in parts)
+
+    @property
+    def is_nested_v1_shape(self) -> bool:
+        return self.message is not None
+
+
+class A2AError(BaseModel):
+    code: int
+    message: str
+
+
+class A2AResponse(BaseModel):
+    jsonrpc: str
+    id: str | None = None
+    result: A2AResult | None = None
+    error: A2AError | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class A2AClient:
+    proxy: ProxyClient
+
+    def register_agent(self, body: AgentRegisterBody) -> Result[AgentResponse]:
+        return self.proxy.transport.post(
+            "/v1/agents",
+            headers=self.proxy.transport.master,
+            json=body,
+            response_type=AgentResponse,
+        )
+
+    def get_agent(self, agent_id: str) -> Result[AgentResponse]:
+        return self.proxy.transport.get(
+            f"/v1/agents/{agent_id}",
+            headers=self.proxy.transport.master,
+            params=NoBody(),
+            response_type=AgentResponse,
+        )
+
+    def delete_agent(self, agent_id: str) -> None:
+        result = self.proxy.transport.delete(
+            f"/v1/agents/{agent_id}",
+            headers=self.proxy.transport.master,
+            json=NoBody(),
+            response_type=NoBody,
+        )
+        if not is_ok(result):
+            warnings.warn(f"delete_agent({agent_id!r}) failed: {result}", stacklevel=2)
+
+    def agent_card(self, agent_id: str, key: str) -> Result[ServedAgentCard]:
+        return self.proxy.transport.get(
+            f"/a2a/{agent_id}/.well-known/agent-card.json",
+            headers=self.proxy.transport.bearer(key),
+            params=NoBody(),
+            response_type=ServedAgentCard,
+        )
+
+    def send_message(self, agent_id: str, key: str, body: A2AJsonRpcRequest) -> Result[A2AResponse]:
+        return self.proxy.transport.post(
+            f"/a2a/{agent_id}",
+            headers=self.proxy.transport.bearer(key),
+            json=body,
+            response_type=A2AResponse,
+        )
+
+
+def build_a2a_client(proxy: ProxyClient) -> A2AClient:
+    return A2AClient(proxy=proxy)
+
+
+def fetch_agent_card(url: str, *, timeout: float = 20.0) -> Result[UpstreamAgentCard]:
+    """Fetch a live A2A agent card from its /.well-known endpoint and parse it into the
+    registration model, so a test can register a real published card verbatim rather
+    than a hand-rolled one."""
+    return get_external(url, response_type=UpstreamAgentCard, timeout=timeout)

tests/e2e/a2a/conftest.py

@@ -0,0 +1,17 @@
+"""A2A suite's `client` fixture.
+
+The shared lifecycle (resources/scoped_key), proxy liveness gate, and e2e marker
+live in the parent tests/e2e/conftest.py. A2AClient holds the shared ProxyClient,
+so the `resources` fixture cleans up keys this suite creates; agents are torn down
+via `resources.defer(...)` in each test.
+"""
+
+import pytest
+
+from a2a_client import A2AClient, build_a2a_client
+from proxy_client import ProxyClient
+
+
+@pytest.fixture(scope="session")
+def client(proxy: ProxyClient) -> A2AClient:
+    return build_a2a_client(proxy)

tests/e2e/a2a/test_a2a_agent_e2e.py

@@ -0,0 +1,202 @@
+"""A2A agents end to end, against a live proxy.
+
+An admin registers an agent whose card pins an A2A protocol version and whose
+litellm_params route it through the completion bridge; a caller then discovers the
+proxy-owned card and drives it over A2A JSON-RPC. These tests assert the recorded
+state (the agent persists, a spend row lands) and the enforced behavior (the served
+card points back at the proxy, message/send returns a real completion in the pinned
+protocol version, and an unsupported version is refused at registration).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from a2a_client import (
+    A2ABridgeParams,
+    A2AClient,
+    A2ADataPart,
+    A2AJsonRpcRequest,
+    A2AMessageSendParams,
+    A2AOutboundMessage,
+    A2ASearchPropertiesParams,
+    A2ASkill,
+    A2ASkillInvocation,
+    A2ATextPart,
+    AgentCardParams,
+    AgentRegisterBody,
+    AgentResponse,
+    fetch_agent_card,
+)
+from e2e_config import unique_marker
+from e2e_http import Result, UnknownApiError, unwrap
+from lifecycle import ResourceManager
+
+BRIDGE = A2ABridgeParams(custom_llm_provider="anthropic", model="claude-haiku-4-5")
+
+MOVEHOME_AGENT_CARD_URL = "https://movehome.org/.well-known/agent.json"
+MOVEHOME_ORIGIN = "https://movehome.org"
+
+pytestmark = pytest.mark.e2e
+
+
+def _register(client: A2AClient, resources: ResourceManager, protocol_version: str) -> AgentResponse:
+    marker = unique_marker()
+    body = AgentRegisterBody(
+        agent_name=f"e2e-a2a-{marker}",
+        agent_card_params=AgentCardParams(
+            protocol_version=protocol_version,
+            name=f"E2E A2A {marker}",
+            description="e2e agent backed by the litellm completion bridge",
+            version="1.0.0",
+            skills=[A2ASkill(id="chat", name="Chat", description="general chat", tags=["chat"])],
+        ),
+        litellm_params=BRIDGE,
+    )
+    agent = unwrap(client.register_agent(body))
+    resources.defer(lambda: client.delete_agent(agent.agent_id))
+    return agent
+
+
+def _register_rejection(client: A2AClient, protocol_version: str) -> Result[AgentResponse]:
+    marker = unique_marker()
+    body = AgentRegisterBody(
+        agent_name=f"e2e-a2a-bad-{marker}",
+        agent_card_params=AgentCardParams(
+            protocol_version=protocol_version,
+            name=f"E2E A2A bad {marker}",
+            description="rejected at registration",
+            version="1.0.0",
+            skills=[A2ASkill(id="chat", name="Chat", description="c", tags=["chat"])],
+        ),
+        litellm_params=BRIDGE,
+    )
+    return client.register_agent(body)
+
+
+def _ask(text: str) -> A2AJsonRpcRequest:
+    return A2AJsonRpcRequest(
+        id=f"e2e-{unique_marker()}",
+        params=A2AMessageSendParams(
+            message=A2AOutboundMessage(parts=[A2ATextPart(text=text)], message_id=unique_marker())
+        ),
+    )
+
+
+class TestA2AAgentLifecycle:
+    @pytest.mark.covers("other.a2a.register.persists")
+    def test_register_persists(self, client: A2AClient, resources: ResourceManager) -> None:
+        agent = _register(client, resources, "0.3")
+        fetched = unwrap(client.get_agent(agent.agent_id))
+        assert fetched.agent_id == agent.agent_id
+        assert fetched.agent_name == agent.agent_name
+        assert fetched.agent_card_params.protocol_version == "0.3"
+
+    @pytest.mark.covers("other.a2a.register.semver_version_accepted")
+    def test_semver_protocol_version_registers_and_serves(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
+        agent = _register(client, resources, "0.3.0")
+        assert agent.agent_card_params.protocol_version == "0.3"
+        card = unwrap(client.agent_card(agent.agent_id, scoped_key))
+        assert card.protocol_version == "0.3"
+        assert card.supported_interfaces is not None
+        assert card.supported_interfaces[0].protocol_version == "0.3"
+        result = unwrap(client.send_message(agent.agent_id, scoped_key, _ask("Say hi in one word"))).result
+        assert result is not None
+        assert result.text != ""
+
+    @pytest.mark.covers("other.a2a.message_send.real_world_agent_replies")
+    def test_real_world_agent_replies_to_property_query(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
+        upstream = unwrap(fetch_agent_card(MOVEHOME_AGENT_CARD_URL)).model_copy(update={"url": MOVEHOME_ORIGIN})
+        assert upstream.protocol_version == "0.3.0"
+        marker = unique_marker()
+        body = AgentRegisterBody(agent_name=f"e2e-a2a-real-{marker}", agent_card_params=upstream)
+        agent = unwrap(client.register_agent(body))
+        resources.defer(lambda: client.delete_agent(agent.agent_id))
+        assert agent.agent_card_params.protocol_version == "0.3"
+        request = A2AJsonRpcRequest(
+            id=f"e2e-{unique_marker()}",
+            params=A2AMessageSendParams(
+                message=A2AOutboundMessage(
+                    parts=[
+                        A2ADataPart(
+                            data=A2ASkillInvocation(
+                                skill="search_properties",
+                                params=A2ASearchPropertiesParams(un_locode="USSFO", service_type="sale", asking_price_max=2_000_000, limit=3),
+                            )
+                        )
+                    ],
+                    message_id=unique_marker(),
+                )
+            ),
+        )
+        response = unwrap(client.send_message(agent.agent_id, scoped_key, request))
+        assert response.error is None
+        assert response.result is not None
+        assert response.result.text.strip() != ""
+
+    @pytest.mark.covers("other.a2a.discovery.proxy_fronted_card")
+    def test_discovery_card_is_proxy_fronted(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
+        agent = _register(client, resources, "0.3")
+        card = unwrap(client.agent_card(agent.agent_id, scoped_key))
+        assert card.url is not None and card.url.endswith(f"/a2a/{agent.agent_id}")
+        assert card.security_schemes is not None
+        scheme = next(iter(card.security_schemes.values()))
+        assert scheme.scheme == "bearer"
+        assert card.supported_interfaces is not None
+        assert card.supported_interfaces[0].url == card.url
+
+    @pytest.mark.covers("other.a2a.message_send.bridge_invokes")
+    def test_message_send_runs_completion_bridge(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
+        agent = _register(client, resources, "0.3")
+        request = _ask("Reply with exactly the word PONG and nothing else")
+        response = unwrap(client.send_message(agent.agent_id, scoped_key, request))
+        assert response.error is None
+        assert response.result is not None
+        assert "PONG" in response.result.text.upper()
+
+        rows = client.proxy.poll_logs_for_request_id(request.id)
+        assert rows, f"no spend log row landed for a2a request {request.id}"
+        assert rows[0].call_type == "asend_message"
+        assert rows[0].model == f"a2a_agent/{agent.agent_card_params.name}"
+
+    @pytest.mark.covers("other.a2a.version.serves_pinned_0_3")
+    def test_pinned_v0_3_serves_flat_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
+        agent = _register(client, resources, "0.3")
+        request = _ask("Say hi in one word")
+        result = unwrap(client.send_message(agent.agent_id, scoped_key, request)).result
+        assert result is not None
+        assert not result.is_nested_v1_shape
+        assert result.kind == "message"
+        assert result.role == "agent"
+        assert result.text != ""
+
+    @pytest.mark.covers("other.a2a.version.serves_pinned_1_0")
+    def test_pinned_v1_0_serves_nested_message_shape(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
+        agent = _register(client, resources, "1.0")
+        request = _ask("Say hi in one word")
+        result = unwrap(client.send_message(agent.agent_id, scoped_key, request)).result
+        assert result is not None
+        assert result.is_nested_v1_shape
+        assert result.message is not None
+        assert result.message.role == "ROLE_AGENT"
+        assert result.text != ""
+
+    @pytest.mark.covers("other.a2a.register.unsupported_version_rejected")
+    def test_unsupported_protocol_version_rejected(self, client: A2AClient) -> None:
+        result = _register_rejection(client, "9.9")
+        match result:
+            case UnknownApiError(status_code=status, body=detail):
+                assert status == 400
+                assert "protocolVersion" in detail
+            case _:
+                pytest.fail(f"expected 400 for unsupported protocolVersion, got {result}")
+
+    @pytest.mark.covers("other.a2a.register.malformed_version_rejected")
+    def test_malformed_protocol_version_rejected(self, client: A2AClient) -> None:
+        result = _register_rejection(client, "0.3.garbage")
+        match result:
+            case UnknownApiError(status_code=status, body=detail):
+                assert status == 400
+                assert "Unsupported protocolVersion '0.3.garbage'" in detail
+            case _:
+                pytest.fail(f"expected 400 for malformed protocolVersion, got {result}")

tests/e2e/batches/batch_client.py

@@ -26,16 +26,24 @@ from e2e_http import (
 )
 from models import LiteLLMParamsBody
 
+UPLOAD_FILENAME = "batch_input.jsonl"
+
 
 class FileObject(BaseModel):
     id: str
     object: str | None = None
     purpose: str | None = None
+    filename: str | None = None
     bytes: int | None = None
     status: str | None = None
     created_at: int | None = None
 
 
+class FileList(BaseModel):
+    object: str | None = None
+    data: list[FileObject] = []
+
+
 class BatchObject(BaseModel):
     id: str
     object: str | None = None
@@ -106,12 +114,30 @@ class BatchClient:
             _files_path(provider),
             headers=self.proxy.transport.bearer(key),
             form=form,
-            filename="batch_input.jsonl",
+            filename=UPLOAD_FILENAME,
             content=content,
             params=ModelQuery(model=model),
             response_type=FileObject,
         )
 
+    def retrieve_file(
+        self, file_id: str, *, key: str, provider: str | None = None
+    ) -> Result[FileObject]:
+        return self.proxy.transport.get(
+            f"{_files_path(provider)}/{file_id}",
+            headers=self.proxy.transport.bearer(key),
+            params=NoBody(),
+            response_type=FileObject,
+        )
+
+    def list_files(self, *, key: str, provider: str | None = None) -> Result[FileList]:
+        return self.proxy.transport.get(
+            _files_path(provider),
+            headers=self.proxy.transport.bearer(key),
+            params=NoBody(),
+            response_type=FileList,
+        )
+
     def create_batch(
         self, *, body: BatchCreateBody, key: str, provider: str | None = None
     ) -> StreamingResponse:

tests/e2e/batches/test_batches_e2e.py

@@ -26,6 +26,7 @@ import pytest
 from e2e_config import require_env, unique_marker
 
 from batch_client import (
+    UPLOAD_FILENAME,
     BatchClient,
     BatchCreateBody,
     BatchObject,
@@ -511,6 +512,72 @@ class TestBatchFileContent:
         )
 
 
+class TestOpenAIFiles:
+    """GET /v1/files (list) and GET /v1/files/{id} (retrieve) over the OpenAI route.
+
+    The proxy lists the OpenAI org's raw file ids, so the list case uploads a raw
+    (provider-routed) file whose id matches what list returns; retrieve re-encodes
+    the id it was called with, so the model-encoded upload round-trips unchanged.
+    """
+
+    @pytest.mark.covers(
+        "llm.files.openai.list.nonstream.works",
+        exercised_on=["files"],
+    )
+    def test_uploaded_file_appears_in_list(
+        self, client: BatchClient, resources: ResourceManager, batch_deployments: None
+    ) -> None:
+        key = resources.key()
+        file = unwrap(
+            client.upload_file(
+                content=render_jsonl(OPENAI_BATCH_MODEL),
+                form=FileUploadForm(purpose="batch"),
+                key=key,
+                provider="openai",
+            )
+        )
+        resources.defer(
+            quietly(lambda: client.delete_file(file.id, key=key, provider="openai"))
+        )
+
+        listed = unwrap(client.list_files(key=key))
+        assert listed.object is None or listed.object == "list", (
+            f"list envelope object={listed.object!r}"
+        )
+        match = next((entry for entry in listed.data if entry.id == file.id), None)
+        assert match is not None, f"uploaded file {file.id!r} absent from GET /v1/files"
+        assert match.purpose == "batch", (
+            f"listed file must round-trip the upload purpose, got {match.purpose!r}"
+        )
+
+    @pytest.mark.covers(
+        "llm.files.openai.retrieve.nonstream.works",
+        exercised_on=["files"],
+    )
+    def test_retrieve_round_trips_metadata(
+        self, client: BatchClient, resources: ResourceManager, batch_deployments: None
+    ) -> None:
+        key = resources.key()
+        file = unwrap(
+            client.upload_file(
+                content=render_jsonl(OPENAI_BATCH_MODEL),
+                form=FileUploadForm(purpose="batch"),
+                model=OPENAI_BATCH_MODEL,
+                key=key,
+            )
+        )
+        resources.defer(quietly(lambda: client.delete_file(file.id, key=key)))
+
+        fetched = unwrap(client.retrieve_file(file.id, key=key))
+        assert fetched.id == file.id, "retrieve must echo the uploaded file id"
+        assert fetched.purpose == "batch", (
+            f"retrieve must round-trip purpose, got {fetched.purpose!r}"
+        )
+        assert fetched.filename == UPLOAD_FILENAME, (
+            f"retrieve must round-trip filename, got {fetched.filename!r}"
+        )
+
+
 BATCH_RL_REQUEST_LINES = 3
 BATCH_RL_RPM_LIMIT = 2
 

tests/e2e/conftest.py

@@ -43,6 +43,10 @@ def pytest_configure(config: pytest.Config) -> None:
         "markers",
         "load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites",
     )
+    config.addinivalue_line(
+        "markers",
+        "weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set",
+    )
 
 
 def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:

tests/e2e/coverage_registry/guardrail.yaml

@@ -31,3 +31,4 @@
 - {id: guardrail.tool_policy.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/tool_policy/tool_policy_guardrail.py", rationale: "Tool-use policy enforcement"}
 - {id: guardrail.mcp_security.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/mcp_security", rationale: "MCP protocol security"}
 - {id: guardrail.llm_as_a_judge.pre_call.blocks, module: guardrail, tier: P2, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/llm_as_a_judge", rationale: "LLM-based judgment guardrail"}
+- {id: guardrail.litellm_content_filter.pre_mcp_call.blocks, module: guardrail, tier: P1, hook_point: pre_mcp_call, assertions: [blocks], exercised_on: [mcp_operations], source: "guardrail_hooks/litellm_content_filter/content_filter.py:_scan_mcp_tool_call_arguments", rationale: "A general content-filter guardrail configured mode=pre_mcp_call blocks a banned keyword in an MCP tool call's arguments before it reaches the upstream MCP server; a clean argument passes"}

tests/e2e/coverage_registry/llm_conversational.yaml

@@ -46,10 +46,10 @@
 - {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"}
 - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven}
 - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (#32831)", fail_before_fix: proven}
-- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: proven}
-- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: proven}
-- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: proven}
-- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: proven}
+- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
+- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
+- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
+- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
 - {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"}
 - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
 - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"}

tests/e2e/coverage_registry/llm_nonconversational.yaml

@@ -32,7 +32,7 @@
 - {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"}
 - {id: llm.rerank.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_rerank_e2e.py:29", rationale: "Cohere rerank, top_n + relevance_score"}
 - {id: llm.files.openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "GET /v1/files/{id}/content returns uploaded batch JSONL bytes"}
-- {id: llm.realtime.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "test_nova_sonic_realtime_e2e.py", rationale: "Nova Sonic realtime session emits response.done (LIT-2239)"}
+- {id: llm.realtime.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "test_realtime_bedrock_e2e.py", rationale: "Nova Sonic realtime session emits response.done (LIT-2239)"}
 - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"}
 - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"}
 - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"}

tests/e2e/coverage_registry/other.yaml

@@ -28,3 +28,12 @@
 - {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"}
 - {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"}
 - {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"}
+- {id: other.a2a.register.persists, module: other, tier: P1, area: a2a, assertions: [persists], source: "agent_endpoints/endpoints.py:325-443", rationale: "POST /v1/agents registers an agent card; GET /v1/agents/{id} reads it back"}
+- {id: other.a2a.register.unsupported_version_rejected, module: other, tier: P1, area: a2a, assertions: [unsupported_version_rejected], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a protocolVersion outside SUPPORTED_A2A_PROTOCOL_VERSIONS is refused with 400"}
+- {id: other.a2a.register.semver_version_accepted, module: other, tier: P1, area: a2a, assertions: [semver_version_accepted], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a patch-level semver like 0.3.0 (what the Google A2A SDK emits) registers, stores and serves the canonical 0.3 rather than 400ing; regression guard for the v1.92 report"}
+- {id: other.a2a.message_send.real_world_agent_replies, module: other, tier: P1, area: a2a, assertions: [real_world_agent_replies], source: "agent_endpoints/a2a_endpoints.py asend_message", rationale: "A real published a2a agent fetched live from a public /.well-known endpoint (pinning the full semver 0.3.0 the a2a-sdk emits) registers, serves the canonical 0.3, and a message/send skill invocation proxies to the live upstream and returns the agent's reply"}
+- {id: other.a2a.register.malformed_version_rejected, module: other, tier: P1, area: a2a, assertions: [malformed_version_rejected], source: "a2a/agent_card.py normalize_protocol_version", rationale: "A malformed protocolVersion like 0.3.garbage fails full-string semver validation and is refused with 400 instead of truncating to a supported family"}
+- {id: other.a2a.discovery.proxy_fronted_card, module: other, tier: P1, area: a2a, assertions: [proxy_fronted_card], source: "agent_endpoints/a2a_endpoints.py get_agent_card", rationale: "/.well-known/agent-card.json serves the proxy url + supportedInterfaces and the LiteLLM virtual-key bearer scheme, not the upstream"}
+- {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"}
+- {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"}
+- {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"}

tests/e2e/coverage_registry/reliability.yaml

@@ -24,3 +24,4 @@
 - {id: reliability.timeout.request_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: request_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions, messages], source: "litellm/router.py:545-551", rationale: "Per-request timeout raises Timeout"}
 - {id: reliability.timeout.stream_timeout.exceeds_deadline, module: reliability, tier: P1, behavior: timeout, variant: stream_timeout, assertions: [exceeds_deadline], exercised_on: [chat_completions], source: "litellm/router.py:551", rationale: "Streaming chunk-delivery timeout"}
 - {id: reliability.perf.throughput.under_slo, module: reliability, tier: P1, behavior: perf, variant: throughput, assertions: [under_slo], exercised_on: [chat_completions, messages], source: grammar, rationale: "Throughput SLO under load"}
+- {id: reliability.perf.session_anomaly.under_slo, module: reliability, tier: P1, behavior: perf, variant: session_anomaly, assertions: [under_slo], exercised_on: [messages], source: grammar, rationale: "Weekly Claude Code-shaped multi-turn session load against real providers; ceilings on error rate, warm-turn cache read/write, p95 turn time, and gateway-recorded spend (LIT-4562)"}

tests/e2e/e2e_config.py

@@ -78,6 +78,24 @@ LOAD_DURATION_SECONDS = float(os.environ.get("E2E_LOAD_DURATION_SECONDS", "60"))
 LOAD_MIN_RPS = float(os.environ.get("E2E_LOAD_MIN_RPS", "355"))
 LOAD_MAX_FAILURE_RATIO = float(os.environ.get("E2E_LOAD_MAX_FAILURE_RATIO", "0.01"))
 
+WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY"
+ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
+ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
+ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))
+ANOMALY_MAX_ERROR_RATIO = float(os.environ.get("E2E_ANOMALY_MAX_ERROR_RATIO", "0.05"))
+ANOMALY_MIN_WARM_CACHE_READ_SHARE = float(
+    os.environ.get("E2E_ANOMALY_MIN_WARM_CACHE_READ_SHARE", "0.65")
+)
+ANOMALY_MAX_P95_TURN_SECONDS = float(
+    os.environ.get("E2E_ANOMALY_MAX_P95_TURN_SECONDS", "30")
+)
+ANOMALY_MAX_KEY_SPEND_USD = float(
+    os.environ.get("E2E_ANOMALY_MAX_KEY_SPEND_USD", "0.60")
+)
+ANOMALY_SPEND_SETTLE_SECONDS = float(
+    os.environ.get("E2E_ANOMALY_SPEND_SETTLE_SECONDS", "75")
+)
+
 
 def require_env(*names: str) -> tuple[str, ...]:
     """Return the non-empty values for each env name, or hard-fail naming which are missing.

tests/e2e/e2e_http.py

@@ -62,6 +62,7 @@ R = TypeVar("R", bound=BaseModel)
 
 class Success(BaseModel, Generic[R]):
     kind: Literal["success"] = "success"
+    status_code: int
     data: R
 
 
@@ -146,6 +147,32 @@ class StreamingResponse(BaseModel):
         return "text/event-stream" in (self.content_type or "")
 
 
+class BinaryStream(BaseModel):
+    """Outcome of consuming a binary chunked response (e.g. TTS audio) as a stream.
+
+    Unlike StreamingResponse, which line-splits an SSE text body, this iterates the
+    raw bytes with iter_content and reports how many non-empty chunks arrived and
+    the total byte count, so a caller can assert customer-observable streaming
+    (multiple chunks, real bytes) without decoding the payload."""
+
+    status_code: int
+    content_type: str | None = None
+    call_id: str | None = None
+    transfer_encoding: str | None = None
+    content_length: str | None = None
+    error_body: str | None = None
+    chunk_count: int = 0
+    total_bytes: int = 0
+
+    @property
+    def ok(self) -> bool:
+        return 200 <= self.status_code < 300
+
+    @property
+    def chunked(self) -> bool:
+        return "chunked" in (self.transfer_encoding or "")
+
+
 def _hdr(resp: requests.Response, name: str) -> str | None:
     value = resp.headers.get(name)
     return value if isinstance(value, str) else None
@@ -159,6 +186,18 @@ def unwrap[R: BaseModel](result: Result[R]) -> R:
             raise AssertionError(result)
 
 
+def unwrap_status[R: BaseModel](result: Result[R], expected_status: int) -> R:
+    """Like unwrap, but also pins the exact HTTP status the success came back on,
+    for routes whose contract is a specific 2xx (e.g. 201 Created on a submission)."""
+    match result:
+        case Success(status_code=status_code, data=data) if status_code == expected_status:
+            return data
+        case Success(status_code=status_code):
+            raise AssertionError(f"expected HTTP {expected_status}, got {status_code}")
+        case _:
+            raise AssertionError(result)
+
+
 def is_ok[R: BaseModel](result: Result[R]) -> bool:
     match result:
         case Success():
@@ -199,7 +238,7 @@ def _classify[R: BaseModel](
     if not resp.ok:
         return UnknownApiError(status_code=resp.status_code, body=resp.text)
     try:
-        return Success(data=response_type.model_validate(resp.json()))
+        return Success(status_code=resp.status_code, data=response_type.model_validate(resp.json()))
     except Exception as exc:  # noqa: BLE001 - any parse/validation failure is a value
         return ValidationError(message=str(exc))
 
@@ -244,6 +283,26 @@ def get[R: BaseModel](
     return _classify(resp, response_type)
 
 
+def get_external[R: BaseModel](
+    url: str,
+    *,
+    response_type: type[R],
+    timeout: float = 30.0,
+) -> Result[R]:
+    """GET an absolute URL outside the proxy (e.g. a public /.well-known document).
+    Unlike the transport wrappers there is no proxy base url and no proxy auth; the
+    response still gets the same tagged-union classification as every other call."""
+    try:
+        resp = requests.get(
+            url,
+            headers={"Accept": "application/json"},
+            timeout=timeout,
+        )
+    except requests.RequestException as exc:
+        return NetworkError(message=str(exc))
+    return _classify(resp, response_type)
+
+
 def delete[R: BaseModel](
     url: URL,
     *,
@@ -286,6 +345,26 @@ def patch[R: BaseModel](
     return _classify(resp, response_type)
 
 
+def put[R: BaseModel](
+    url: URL,
+    *,
+    headers: BaseModel,
+    json: BaseModel,
+    response_type: type[R],
+    timeout: float = 30.0,
+) -> Result[R]:
+    try:
+        resp = requests.put(
+            str(url),
+            headers=_headers(headers),
+            json=json.model_dump(by_alias=True, exclude_none=True),
+            timeout=timeout,
+        )
+    except requests.RequestException as exc:
+        return NetworkError(message=str(exc))
+    return _classify(resp, response_type)
+
+
 def probe(
     url: URL, *, headers: BaseModel, params: BaseModel, timeout: float = 30.0
 ) -> ProbeResult:
@@ -397,16 +476,18 @@ def upload[R: BaseModel](
     url: URL,
     *,
     headers: BaseModel,
-    form: FileUploadForm,
+    form: BaseModel,
     filename: str,
     content: bytes,
+    file_content_type: str = "application/jsonl",
     params: BaseModel | None = None,
     response_type: type[R],
     timeout: float = 60.0,
 ) -> Result[R]:
-    """Multipart POST for file uploads (/v1/files). Form fields come from `form`,
-    the file bytes are sent as the `file` part, and `params` carries any query
-    routing (e.g. ?model=). requests sets the multipart Content-Type itself."""
+    """Multipart POST for file-bearing routes (/v1/files, /v1/audio/transcriptions).
+    Form fields come from `form`, the file bytes are sent as the `file` part with
+    `file_content_type`, and `params` carries any query routing (e.g. ?model=).
+    requests sets the multipart Content-Type itself."""
     dumped: dict[str, object] = form.model_dump(by_alias=True, exclude_none=True)
     data = {key: str(value) for key, value in dumped.items()}
     try:
@@ -415,7 +496,7 @@ def upload[R: BaseModel](
             headers=_headers(headers),
             params=_params(params),
             data=data,
-            files={"file": (filename, content, "application/jsonl")},
+            files={"file": (filename, content, file_content_type)},
             timeout=timeout,
         )
     except requests.RequestException as exc:
@@ -423,6 +504,54 @@ def upload[R: BaseModel](
     return _classify(resp, response_type)
 
 
+def stream_binary(
+    url: URL,
+    *,
+    headers: BaseModel,
+    json: BaseModel,
+    chunk_size: int = 8192,
+    timeout: float = 60.0,
+) -> BinaryStream:
+    """POST that consumes a binary chunked response (e.g. TTS audio) as a stream,
+    counting non-empty chunks and total bytes with iter_content. A non-2xx status
+    short-circuits with the counts left at zero so the caller can fail loudly."""
+    try:
+        resp = requests.post(
+            str(url),
+            headers=_headers(headers),
+            json=json.model_dump(by_alias=True, exclude_none=True),
+            stream=True,
+            timeout=timeout,
+        )
+    except requests.RequestException as exc:
+        return BinaryStream(status_code=-1, error_body=str(exc)[:300])
+    with resp:
+        content_type = _hdr(resp, "content-type")
+        call_id = _hdr(resp, "x-litellm-call-id")
+        transfer_encoding = _hdr(resp, "transfer-encoding")
+        content_length = _hdr(resp, "content-length")
+        if not (200 <= resp.status_code < 300):
+            return BinaryStream(
+                status_code=resp.status_code,
+                content_type=content_type,
+                call_id=call_id,
+                transfer_encoding=transfer_encoding,
+                content_length=content_length,
+                error_body=resp.text[:300],
+            )
+        raw_chunks = cast("Iterator[bytes]", resp.iter_content(chunk_size=chunk_size))
+        chunks = tuple(chunk for chunk in raw_chunks if chunk)
+        return BinaryStream(
+            status_code=resp.status_code,
+            content_type=content_type,
+            call_id=call_id,
+            transfer_encoding=transfer_encoding,
+            content_length=content_length,
+            chunk_count=len(chunks),
+            total_bytes=sum(len(chunk) for chunk in chunks),
+        )
+
+
 def download(
     url: URL, *, headers: BaseModel, timeout: float = 60.0
 ) -> StreamingResponse:

tests/e2e/guardrails/guardrails_client.py

@@ -10,13 +10,15 @@ from typing import Literal
 
 from pydantic import BaseModel
 
-from e2e_config import POLL_INTERVAL, POLL_TIMEOUT
+from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker
 from e2e_http import NoBody, Result, Success, unwrap
+from lifecycle import ResourceManager
 from models import (
     ChatBody,
     ChatMessage,
     ChatResponse,
     KeyGenerateBody,
+    LiteLLMParamsBody,
     TeamDeleteBody,
     TeamInfoParams,
     TeamInfoResponse,
@@ -54,7 +56,33 @@ class BedrockGuardrailParamsBody(GuardrailParamsBase):
     aws_region_name: str | None = None
 
 
-GuardrailParamsBody = ContentFilterParamsBody | BedrockGuardrailParamsBody
+class OpenAIModerationParamsBody(GuardrailParamsBase):
+    guardrail: Literal["openai_moderation"] = "openai_moderation"
+    api_key: str | None = None
+    model: str | None = None
+
+
+class PresidioParamsBody(GuardrailParamsBase):
+    guardrail: Literal["presidio"] = "presidio"
+    presidio_analyzer_api_base: str | None = None
+    presidio_anonymizer_api_base: str | None = None
+    # apply_to_output masks PII the model itself emitted, which also makes the
+    # guardrail run post_call. logging_only masks what the proxy logs.
+    apply_to_output: bool | None = None
+    logging_only: bool | None = None
+
+
+class BlockCodeExecutionParamsBody(GuardrailParamsBase):
+    guardrail: Literal["block_code_execution"] = "block_code_execution"
+
+
+GuardrailParamsBody = (
+    ContentFilterParamsBody
+    | BedrockGuardrailParamsBody
+    | OpenAIModerationParamsBody
+    | PresidioParamsBody
+    | BlockCodeExecutionParamsBody
+)
 
 
 class GuardrailSpecBody(BaseModel):
@@ -135,6 +163,35 @@ class GuardrailsClient:
             )
         ).guardrail_id
 
+    def create_backend_model(self, resources: ResourceManager, prefix: str = "e2e-guard-backend") -> str:
+        """Register a gemini chat deployment for a guardrail test to run against
+        (deleted on teardown). The guardrails under test here gate on prompt/output
+        content, not the backend, so a single cheap deployment stands in for the
+        model the customer would call."""
+        model_name = f"{prefix}-{unique_marker()}"
+        model_id = self.proxy.create_model(
+            model_name,
+            LiteLLMParamsBody(model="gemini/gemini-2.5-flash", api_key="os.environ/GEMINI_API_KEY"),
+        )
+        resources.defer(lambda: self.proxy.delete_model(model_id))
+        return model_name
+
+    def register(self, name: str, params: GuardrailParamsBody) -> str:
+        """Register any guardrail via POST /guardrails and return its id. New
+        built-ins register with default_on=False and are opted into per request
+        via the chat body's `guardrails` list, so one guardrail under test never
+        intercepts unrelated traffic on the shared proxy."""
+        return unwrap(
+            self.proxy.transport.post(
+                "/guardrails",
+                headers=self.proxy.transport.master,
+                json=GuardrailCreateBody(
+                    guardrail=GuardrailSpecBody(guardrail_name=name, litellm_params=params)
+                ),
+                response_type=GuardrailCreateResponse,
+            )
+        ).guardrail_id
+
     def delete_guardrail(self, guardrail_id: str) -> None:
         _ = self.proxy.transport.delete(
             f"/guardrails/{guardrail_id}",
@@ -171,13 +228,27 @@ class GuardrailsClient:
             KeyGenerateBody(team_id=team_id, user_id="e2e-guardrails-user")
         )
 
-    def chat(self, key: str, model: str, text: str) -> Result[ChatResponse]:
+    def chat(
+        self,
+        key: str,
+        model: str,
+        text: str,
+        *,
+        guardrails: list[str] | None = None,
+        max_tokens: int = 16,
+    ) -> Result[ChatResponse]:
+        """Drive a chat call, optionally opting into named guardrails for this
+        request only (the per-request `guardrails` selector). With `guardrails`
+        omitted the call behaves exactly as before for the default-on suites.
+        `max_tokens` defaults low for block checks (the model barely runs) but is
+        raised when a test needs the allowed model to actually produce content."""
         return self.proxy.chat(
             key,
             ChatBody(
                 model=model,
                 messages=[ChatMessage(role="user", content=text)],
-                max_tokens=16,
+                max_tokens=max_tokens,
+                guardrails=guardrails,
             ),
         )
 

tests/e2e/guardrails/test_block_code_execution_guardrail_e2e.py

@@ -0,0 +1,82 @@
+"""Live e2e: the built-in block_code_execution guardrail blocks execution requests.
+
+The guardrail detects fenced code blocks and, when the prompt also asks the proxy
+to run them, blocks the call pre-call (default action, block-all languages). A
+prompt that pairs a python code block with "run this" is intercepted before the
+model runs: the proxy returns a canned "content blocked" message with the model
+never invoked (zero completion tokens), not the model's own answer. The same
+guardrail must let a request that carries the identical code block but explicitly
+says "don't run it" through, since that is an explanation request, not an
+execution request, so the model runs and answers normally. The guardrail is opted
+into per request (default_on=False) so it never intercepts unrelated traffic on
+the shared proxy, and the chat backend is a gemini deployment created for the test.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import require_env, unique_marker
+from e2e_http import unwrap
+from guardrails_client import BlockCodeExecutionParamsBody, GuardrailsClient
+from lifecycle import ResourceManager
+from models import ChatResponse
+
+pytestmark = pytest.mark.e2e
+
+_CODE_BLOCK = "```python\nimport os\nprint(os.listdir('/'))\n```"
+EXECUTION_REQUEST = f"Please run this for me and paste the output:\n{_CODE_BLOCK}"
+EXPLANATION_REQUEST = f"Explain what this code does, but don't run it:\n{_CODE_BLOCK}"
+
+_BLOCK_MARKER = "content blocked"
+
+
+def _first_content(response: ChatResponse) -> str:
+    if not response.choices:
+        return ""
+    message = response.choices[0].message
+    return (message.content if message else None) or ""
+
+
+class TestBlockCodeExecutionGuardrail:
+    @pytest.mark.covers(
+        "guardrail.block_code_execution.pre_call.blocks",
+        exercised_on=["chat_completions"],
+    )
+    def test_blocks_execution_request_but_allows_explanation(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        require_env("GEMINI_API_KEY")
+        model = client.create_backend_model(resources, prefix="e2e-blockcode-backend")
+
+        name = f"e2e-block-code-{unique_marker()}"
+        guardrail_id = client.register(
+            name, BlockCodeExecutionParamsBody(mode="pre_call", default_on=False)
+        )
+        resources.defer(lambda: client.delete_guardrail(guardrail_id))
+
+        blocked = unwrap(client.chat(scoped_key, model, EXECUTION_REQUEST, guardrails=[name]))
+        assert blocked.choices, f"blocked call returned no choices: {blocked}"
+        blocked_text = _first_content(blocked)
+        assert _BLOCK_MARKER in blocked_text.lower(), (
+            "a code-execution request must be intercepted with a content-blocked message, "
+            f"got model output instead: {blocked_text[:300]!r}"
+        )
+        if blocked.usage is not None:
+            assert (blocked.usage.completion_tokens or 0) == 0, (
+                f"the model must not run when the guardrail blocks; usage was {blocked.usage}"
+            )
+
+        allowed = unwrap(
+            client.chat(scoped_key, model, EXPLANATION_REQUEST, guardrails=[name], max_tokens=256)
+        )
+        allowed_text = _first_content(allowed)
+        assert _BLOCK_MARKER not in allowed_text.lower(), (
+            "an explanation request that says 'don't run it' must not be blocked, but got the "
+            f"content-blocked message: {allowed_text[:300]!r}"
+        )
+        ran = allowed.usage is not None and (allowed.usage.prompt_tokens or 0) > 0
+        assert ran, (
+            "the explanation request must reach the model (the guardrail lets it through), but "
+            f"the model was never invoked; usage was {allowed.usage}"
+        )

tests/e2e/guardrails/test_openai_moderation_guardrail_e2e.py

@@ -0,0 +1,66 @@
+"""Live e2e: the built-in OpenAI moderation guardrail blocks flagged input pre-call.
+
+Registers an `openai_moderation` guardrail (default_on=False) and opts into it for
+one request via the chat body's `guardrails` selector, so it never intercepts
+unrelated traffic on the shared proxy. The guardrail calls OpenAI's moderation API
+before the upstream model runs; a prompt that trips the policy must be rejected
+with HTTP 400 naming the moderation policy, and the same guardrail must let a
+benign prompt through. The chat backend is a gemini deployment created for the
+test (and torn down); moderation runs independently of it, so the block is
+attributable to the guardrail, not the model.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import require_env, unique_marker
+from e2e_http import UnknownApiError, unwrap
+from guardrails_client import GuardrailsClient, OpenAIModerationParamsBody
+from lifecycle import ResourceManager
+
+pytestmark = pytest.mark.e2e
+
+# A direct threat of violence reliably trips OpenAI's harassment/violence categories.
+FLAGGED_PROMPT = "I am going to find you and violently kill you and your entire family tonight."
+BENIGN_PROMPT = "Reply with the single word: hello"
+
+
+class TestOpenAIModerationGuardrail:
+    @pytest.mark.covers(
+        "guardrail.openai_moderations.pre_call.blocks",
+        exercised_on=["chat_completions"],
+    )
+    def test_moderation_blocks_flagged_input(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        require_env("OPENAI_API_KEY", "GEMINI_API_KEY")
+        model = client.create_backend_model(resources, prefix="e2e-moderation-backend")
+
+        name = f"e2e-openai-moderation-{unique_marker()}"
+        guardrail_id = client.register(
+            name,
+            OpenAIModerationParamsBody(
+                mode="pre_call", default_on=False, api_key="os.environ/OPENAI_API_KEY"
+            ),
+        )
+        resources.defer(lambda: client.delete_guardrail(guardrail_id))
+
+        blocked = client.chat(scoped_key, model, FLAGGED_PROMPT, guardrails=[name])
+        match blocked:
+            case UnknownApiError(status_code=400, body=body):
+                assert "moderation" in body.lower(), (
+                    f"the block body must name the moderation policy, got: {body[:400]}"
+                )
+            case UnknownApiError(status_code=status, body=body):
+                pytest.fail(f"expected a 400 moderation block, got {status}: {body[:400]}")
+            case _:
+                pytest.fail(
+                    f"openai moderation did not block a flagged prompt; got {blocked}"
+                )
+
+        allowed = unwrap(client.chat(scoped_key, model, BENIGN_PROMPT, guardrails=[name]))
+        assert allowed.choices, (
+            "the same moderation guardrail must let a benign prompt through, but the "
+            f"call returned no choices: {allowed}"
+        )

tests/e2e/guardrails/test_presidio_guardrail_e2e.py

@@ -0,0 +1,211 @@
+"""Live e2e: the built-in Presidio PII guardrail masks PII on the request, on the
+model output, and in what the proxy logs.
+
+Presidio replaces detected PII with `<ENTITY_TYPE>` placeholders (e.g.
+`<EMAIL_ADDRESS>`) via a real analyzer + anonymizer. Three modes are checked
+independently, each opted into per request (default_on=False) so it never touches
+unrelated traffic:
+
+- pre_call: the prompt is anonymized before it reaches the model, so a
+  repeat-verbatim request comes back with the placeholder, never the raw email
+- post_call (apply_to_output): PII the model itself emits is masked on the way
+  out, so the caller never receives the raw value the model produced
+- logging_only: the call is not blocked, and the request the proxy records is
+  masked. That is read back from the real OTEL destination (Jaeger): the gen-AI
+  span's `gen_ai.input.messages` attribute carries the masked placeholder, never
+  the raw email
+
+Analyzer/anonymizer bases come from PRESIDIO_ANALYZER_API_BASE /
+PRESIDIO_ANONYMIZER_API_BASE (compose provides the in-network hosts; point them at
+locally published container ports for a host run). The logging_only check needs
+the OTEL v2 logger active and its destination readable at OTEL_QUERY_URL, with
+message-content capture on (OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT).
+The chat backend is a gemini deployment created for the test.
+"""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+
+from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, require_env, unique_marker
+from e2e_http import NoBody, require_successful_call, unwrap
+from guardrails_client import GuardrailMode, GuardrailsClient, PresidioParamsBody
+from lifecycle import ResourceManager
+from models import ChatBody, ChatMessage, ChatResponse, ReadinessDetailsResponse
+from otel_client import JaegerSpan, OtelReader, build_otel_reader
+
+pytestmark = pytest.mark.e2e
+
+RAW_EMAIL = "alice.example.person@example.com"
+PLACEHOLDER = "<EMAIL_ADDRESS>"
+
+ECHO_REQUEST = f"Repeat the following text back exactly, verbatim, with no changes: My email is {RAW_EMAIL}"
+EMIT_REQUEST = f"Output exactly this one line and nothing else: Please contact {RAW_EMAIL} today"
+LOG_REQUEST = f"Say hello and include this email once verbatim: {RAW_EMAIL}"
+
+OTEL_V2_LOGGER = "OpenTelemetryV2"
+INPUT_MESSAGES_TAG = "gen_ai.input.messages"
+
+
+def _content(response: ChatResponse) -> str:
+    if not response.choices:
+        return ""
+    message = response.choices[0].message
+    return (message.content if message else None) or ""
+
+
+def _span_tag(span: JaegerSpan, key: str) -> str | None:
+    for tag in span.tags:
+        if tag.key == key and isinstance(tag.value, str):
+            return tag.value
+    return None
+
+
+def _poll_logged_prompt(reader: OtelReader, *, call_id: str, genai_span: str) -> str | None:
+    """Poll the OTEL destination until the call's gen-AI span carries a masked
+    logged prompt, and return it. logging_only masks the payload asynchronously,
+    so the span can briefly export before the mask lands; polling to a deadline
+    waits that out and returns the last value seen so the caller's assertions
+    report the real final state if it never masks."""
+    deadline = time.monotonic() + POLL_TIMEOUT
+    last: str | None = None
+    while time.monotonic() < deadline:
+        for trace in reader.traces_for_call(call_id):
+            for span in trace.spans:
+                if span.operation_name != genai_span:
+                    continue
+                value = _span_tag(span, INPUT_MESSAGES_TAG)
+                if value is not None:
+                    last = value
+                    if PLACEHOLDER in value and RAW_EMAIL not in value:
+                        return value
+        time.sleep(POLL_INTERVAL)
+    return last
+
+
+def _presidio_params(
+    mode: GuardrailMode, *, apply_to_output: bool = False, logging_only: bool = False
+) -> PresidioParamsBody:
+    analyzer, anonymizer = require_env(
+        "PRESIDIO_ANALYZER_API_BASE", "PRESIDIO_ANONYMIZER_API_BASE"
+    )
+    return PresidioParamsBody(
+        mode=mode,
+        default_on=False,
+        presidio_analyzer_api_base=analyzer,
+        presidio_anonymizer_api_base=anonymizer,
+        apply_to_output=apply_to_output,
+        logging_only=logging_only,
+    )
+
+
+def _require_otel_v2_active(client: GuardrailsClient) -> None:
+    details = unwrap(
+        client.proxy.transport.get(
+            "/health/readiness/details",
+            headers=client.proxy.transport.master,
+            params=NoBody(),
+            response_type=ReadinessDetailsResponse,
+        )
+    )
+    assert OTEL_V2_LOGGER in details.success_callbacks, (
+        f"the logging_only check reads the masked prompt back from OTEL, so the proxy must have "
+        f"the {OTEL_V2_LOGGER} logger active; got callbacks: {details.success_callbacks}"
+    )
+
+
+class TestPresidioGuardrail:
+    @pytest.mark.covers(
+        "guardrail.presidio.pre_call.masks",
+        exercised_on=["chat_completions"],
+    )
+    def test_pre_call_masks_pii_before_the_model_sees_it(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        require_env("GEMINI_API_KEY")
+        model = client.create_backend_model(resources, prefix="e2e-presidio-pre")
+        name = f"e2e-presidio-pre-{unique_marker()}"
+        guardrail_id = client.register(name, _presidio_params("pre_call"))
+        resources.defer(lambda: client.delete_guardrail(guardrail_id))
+
+        echoed = _content(
+            unwrap(client.chat(scoped_key, model, ECHO_REQUEST, guardrails=[name], max_tokens=128))
+        )
+        assert RAW_EMAIL not in echoed, (
+            "pre_call masking must strip the raw email before the model sees it, but the "
+            f"model echoed it back: {echoed[:300]!r}"
+        )
+        assert PLACEHOLDER in echoed, (
+            "the model should have echoed the masked placeholder the guardrail substituted, "
+            f"got: {echoed[:300]!r}"
+        )
+
+    @pytest.mark.covers(
+        "guardrail.presidio.post_call.masks",
+        exercised_on=["chat_completions"],
+    )
+    def test_post_call_masks_pii_in_model_output(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        require_env("GEMINI_API_KEY")
+        model = client.create_backend_model(resources, prefix="e2e-presidio-post")
+        name = f"e2e-presidio-post-{unique_marker()}"
+        guardrail_id = client.register(name, _presidio_params("post_call", apply_to_output=True))
+        resources.defer(lambda: client.delete_guardrail(guardrail_id))
+
+        out = _content(
+            unwrap(client.chat(scoped_key, model, EMIT_REQUEST, guardrails=[name], max_tokens=128))
+        )
+        assert RAW_EMAIL not in out, (
+            "post_call masking must strip PII the model emitted, but the raw email reached the "
+            f"caller: {out[:300]!r}"
+        )
+        assert PLACEHOLDER in out, (
+            f"the masked placeholder should replace the model's PII output, got: {out[:300]!r}"
+        )
+
+    @pytest.mark.covers(
+        "guardrail.presidio.logging_only.masks",
+        exercised_on=["chat_completions"],
+    )
+    def test_logging_only_masks_the_logged_prompt(
+        self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
+    ) -> None:
+        require_env("GEMINI_API_KEY")
+        _require_otel_v2_active(client)
+        reader = build_otel_reader()
+
+        model = client.create_backend_model(resources, prefix="e2e-presidio-log")
+        name = f"e2e-presidio-log-{unique_marker()}"
+        guardrail_id = client.register(name, _presidio_params("logging_only", logging_only=True))
+        resources.defer(lambda: client.delete_guardrail(guardrail_id))
+
+        outcome = client.proxy.transport.send(
+            "/chat/completions",
+            headers=client.proxy.transport.bearer(scoped_key),
+            json=ChatBody(
+                model=model,
+                messages=[ChatMessage(role="user", content=LOG_REQUEST)],
+                max_tokens=64,
+                guardrails=[name],
+            ),
+        )
+        require_successful_call(outcome)  # logging_only must not block
+        assert outcome.call_id is not None, "the response must carry x-litellm-call-id to find its trace"
+
+        genai_span = f"chat {model}"
+        logged_prompt = _poll_logged_prompt(reader, call_id=outcome.call_id, genai_span=genai_span)
+        assert logged_prompt is not None, (
+            f"the gen-AI span {genai_span!r} never recorded {INPUT_MESSAGES_TAG} at the OTEL "
+            "destination within the deadline (message-content capture must be on, and the trace "
+            "must reach the destination)"
+        )
+        assert RAW_EMAIL not in logged_prompt, (
+            "logging_only must mask the PII the proxy records for the request, but the raw email "
+            f"is present in the logged prompt: {logged_prompt[:400]!r}"
+        )
+        assert PLACEHOLDER in logged_prompt, (
+            f"the logged prompt must carry the masked placeholder, got: {logged_prompt[:400]!r}"
+        )

tests/e2e/guardrails/test_team_disable_global_guardrail_e2e.py

@@ -8,6 +8,8 @@ suite was removed.
 
 from __future__ import annotations
 
+import time
+
 import pytest
 
 from e2e_config import unique_marker
@@ -19,11 +21,39 @@ pytestmark = pytest.mark.e2e
 
 MODEL = "gemini-2.5-flash"
 
+# A guardrail created via POST /guardrails is registered in-process immediately
+# on the worker that served the create call, but the proxy runs multiple
+# pods/workers behind the shared key, and every other one only picks up the new
+# guardrail on its next periodic DB sync (every 30s), so the very next request
+# can race a worker that has not synced yet.
+GUARDRAIL_PROPAGATION_DEADLINE_SECONDS = 40.0
+GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0
+
 
 def _prompt_with(banned_keyword: str) -> str:
     return f"Reply with the single word OK. {banned_keyword}"
 
 
+def _assert_eventually_blocked(client: GuardrailsClient, key: str, banned: str) -> None:
+    deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
+    while True:
+        result = client.chat(key, MODEL, _prompt_with(banned))
+        match result:
+            case UnknownApiError(status_code=status, body=body):
+                assert status == 400, f"expected a 400 guardrail block, got {status}: {body[:300]}"
+                assert "content blocked" in body.lower() or banned in body, (
+                    f"block response missing content-filter reason: {body[:300]}"
+                )
+                return
+            case _ if time.monotonic() < deadline:
+                time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
+            case _:
+                pytest.fail(
+                    f"default-on guardrail never blocked the banned keyword within "
+                    f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; got {result}"
+                )
+
+
 class TestTeamDisableGlobalGuardrail:
     @pytest.mark.covers(
         "guardrail.litellm_content_filter.pre_call.blocks",
@@ -33,25 +63,10 @@ class TestTeamDisableGlobalGuardrail:
         self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
     ) -> None:
         banned = unique_marker()
-        guardrail_id = client.create_content_filter_guardrail(
-            f"e2e-content-filter-{banned}", banned
-        )
+        guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned)
         resources.defer(lambda: client.delete_guardrail(guardrail_id))
 
-        result = client.chat(scoped_key, MODEL, _prompt_with(banned))
-
-        match result:
-            case UnknownApiError(status_code=status, body=body):
-                assert status == 400, (
-                    f"expected a 400 guardrail block, got {status}: {body[:300]}"
-                )
-                assert "content blocked" in body.lower() or banned in body, (
-                    f"block response missing content-filter reason: {body[:300]}"
-                )
-            case _:
-                pytest.fail(
-                    f"default-on guardrail did not block the banned keyword; got {result}"
-                )
+        _assert_eventually_blocked(client, scoped_key, banned)
 
     @pytest.mark.covers(
         "guardrail.litellm_content_filter.pre_call.allows",
@@ -61,14 +76,10 @@ class TestTeamDisableGlobalGuardrail:
         self, client: GuardrailsClient, resources: ResourceManager
     ) -> None:
         banned = unique_marker()
-        guardrail_id = client.create_content_filter_guardrail(
-            f"e2e-content-filter-{banned}", banned
-        )
+        guardrail_id = client.create_content_filter_guardrail(f"e2e-content-filter-{banned}", banned)
         resources.defer(lambda: client.delete_guardrail(guardrail_id))
 
-        team_id = client.create_team_opted_out_of_global_guardrails(
-            f"e2e-guardrail-optout-{banned}"
-        )
+        team_id = client.create_team_opted_out_of_global_guardrails(f"e2e-guardrail-optout-{banned}")
         resources.defer(lambda: client.delete_team(team_id))
         key = client.create_key_in_team(team_id)
         resources.defer(lambda: client.proxy.delete_key(key))

tests/e2e/llm_translation/endpoints_client.py

@@ -15,7 +15,7 @@ from typing import Literal
 from pydantic import BaseModel
 
 from proxy_client import ProxyClient
-from e2e_http import StreamingResponse
+from e2e_http import BinaryStream, Result, StreamingResponse
 from models import CacheControl, ChatMessage, LiteLLMParamsBody, RichMessage, TextBlock
 
 __all__ = [
@@ -110,6 +110,16 @@ class ImageRequest(BaseModel):
     size: str = "1024x1024"
 
 
+class TranscriptionForm(BaseModel):
+    model: str
+    response_format: str = "json"
+
+
+class ModerationRequest(BaseModel):
+    model: str
+    input: str
+
+
 class ResponsesOutputContent(BaseModel):
     type: str | None = None
     text: str | None = None
@@ -213,6 +223,27 @@ class ImagesResult(BaseModel):
     data: list[ImageItem] = []
 
 
+class TranscriptionResult(BaseModel):
+    text: str = ""
+
+
+class ModerationResultItem(BaseModel):
+    flagged: bool
+    categories: dict[str, bool] = {}
+
+    @property
+    def flagged_categories(self) -> tuple[str, ...]:
+        return tuple(name for name, hit in self.categories.items() if hit)
+
+
+class ModerationResult(BaseModel):
+    results: list[ModerationResultItem] = []
+
+    @property
+    def first(self) -> ModerationResultItem | None:
+        return self.results[0] if self.results else None
+
+
 @dataclass(frozen=True, slots=True)
 class EndpointsClient:
     proxy: ProxyClient
@@ -314,6 +345,36 @@ class EndpointsClient:
             "/v1/audio/speech", key, SpeechRequest(model=model, input=text, voice=voice)
         )
 
+    def audio_speech_stream(
+        self, key: str, model: str, text: str, *, voice: str = "alloy"
+    ) -> BinaryStream:
+        return self.proxy.transport.stream_binary(
+            "/v1/audio/speech",
+            headers=self.proxy.transport.bearer(key),
+            json=SpeechRequest(model=model, input=text, voice=voice),
+        )
+
+    def transcribe(
+        self, key: str, model: str, *, filename: str, content: bytes
+    ) -> Result[TranscriptionResult]:
+        return self.proxy.transport.upload(
+            "/v1/audio/transcriptions",
+            headers=self.proxy.transport.bearer(key),
+            form=TranscriptionForm(model=model),
+            filename=filename,
+            content=content,
+            file_content_type="audio/wav",
+            response_type=TranscriptionResult,
+        )
+
+    def moderations(self, key: str, model: str, text: str) -> Result[ModerationResult]:
+        return self.proxy.transport.post(
+            "/v1/moderations",
+            headers=self.proxy.transport.bearer(key),
+            json=ModerationRequest(model=model, input=text),
+            response_type=ModerationResult,
+        )
+
     def images(self, key: str, model: str, prompt: str) -> StreamingResponse:
         return self._send(
             "/v1/images/generations", key, ImageRequest(model=model, prompt=prompt)

tests/e2e/llm_translation/test_audio_speech_e2e.py

@@ -1,8 +1,9 @@
-"""Live e2e: POST /v1/audio/speech returns audio.
+"""Live e2e: POST /v1/audio/speech returns audio, non-streamed and streamed.
 
-Registers an OpenAI text-to-speech deployment at runtime and asserts the response
-is an audio body (binary, not JSON). Migrated from
-litellm-regression-tests/tests/test_inference_endpoints.py.
+The non-streamed call asserts an audio (not JSON) body. The streamed call consumes
+the response the way a player would and asserts customer-observable streaming:
+chunked transfer encoding (a buffered body would carry a content-length) with
+non-zero audio bytes.
 """
 
 from __future__ import annotations
@@ -19,6 +20,7 @@ pytestmark = pytest.mark.e2e
 
 
 class TestAudioSpeech:
+    @pytest.mark.covers("llm.audio_speech.openai.basic.nonstream.works")
     def test_audio_speech_returns_audio(
         self, endpoints_client: EndpointsClient, resources: ResourceManager
     ) -> None:
@@ -38,3 +40,39 @@ class TestAudioSpeech:
             f"/audio/speech content-type is not audio: {result.content_type!r}"
         )
         assert result.body, "/audio/speech returned an empty body"
+
+    @pytest.mark.covers("llm.audio_speech.openai.basic.stream.works")
+    def test_audio_speech_streams_audio_chunks(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = f"e2e-speech-stream-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY"
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.audio_speech_stream(
+            key,
+            model,
+            "Streaming speech should arrive in several audio chunks so a client can "
+            "begin playback well before the whole clip has finished generating.",
+        )
+        assert result.ok, (
+            f"/audio/speech stream failed (status {result.status_code}); body={result.error_body}"
+        )
+        assert "audio" in (result.content_type or ""), (
+            f"/audio/speech content-type is not audio: {result.content_type!r}"
+        )
+        assert result.chunked, (
+            f"/audio/speech did not stream: transfer-encoding={result.transfer_encoding!r}, "
+            f"content-length={result.content_length!r} (a buffered body is not a stream)"
+        )
+        assert result.content_length is None, (
+            f"/audio/speech advertised content-length={result.content_length!r} on a "
+            f"streamed response (a buffered body is not a stream)"
+        )
+        assert result.total_bytes > 0, "/audio/speech stream returned no audio bytes"

tests/e2e/llm_translation/test_audio_transcriptions_e2e.py

@@ -0,0 +1,51 @@
+"""Live e2e: POST /v1/audio/transcriptions turns speech into text.
+
+Registers an OpenAI speech-to-text deployment at runtime and uploads a spoken
+weather question (the realtime suite's 24kHz WAV fixture) as multipart, asserting
+the returned transcript is non-empty and mentions the word it was asked about.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from e2e_config import unique_marker
+from e2e_http import unwrap
+from endpoints_client import EndpointsClient
+from lifecycle import ResourceManager
+from models import LiteLLMParamsBody
+
+pytestmark = pytest.mark.e2e
+
+WEATHER_WAV = (
+    Path(__file__).resolve().parent / "realtime" / "fixtures" / "weather_question_24k.wav"
+)
+
+
+class TestAudioTranscriptions:
+    @pytest.mark.covers("llm.audio_transcriptions.openai.basic.nonstream.works")
+    def test_audio_transcriptions_returns_text(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = f"e2e-transcribe-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="openai/gpt-4o-mini-transcribe", api_key="os.environ/OPENAI_API_KEY"
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = unwrap(
+            endpoints_client.transcribe(
+                key, model, filename=WEATHER_WAV.name, content=WEATHER_WAV.read_bytes()
+            )
+        )
+        text = result.text.strip()
+        assert text, "/audio/transcriptions returned an empty transcript"
+        assert "weather" in text.lower(), (
+            f"transcript of a spoken weather question does not mention weather: {text!r}"
+        )

tests/e2e/llm_translation/test_chat_completions_regression_e2e.py

@@ -19,17 +19,176 @@ from __future__ import annotations
 import os
 
 import pytest
+from pydantic import BaseModel
 
 from e2e_config import require_env, unique_marker
-from e2e_http import unwrap
+from e2e_http import StreamingResponse, unwrap
 from lifecycle import ResourceManager
-from models import ChatBody, ChatMessage, LiteLLMParamsBody
+from models import (
+    ChatBody,
+    ChatMessage,
+    ChatResponse,
+    ChatTool,
+    ChatToolFunction,
+    ImageContentPart,
+    ImageUrl,
+    LiteLLMParamsBody,
+    TextContentPart,
+    ThinkingParam,
+)
 from passthrough_client import PassthroughClient
 
 pytestmark = pytest.mark.e2e
 
 COHERE_BACKEND = "cohere/command-r-08-2024"
 GEMINI_BACKEND = "gemini/gemini-2.5-flash"
+OPENAI_BACKEND = "openai/gpt-5.6"
+BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+
+
+class _StreamToolCallFunction(BaseModel):
+    name: str | None = None
+    arguments: str | None = None
+
+
+class _StreamToolCall(BaseModel):
+    function: _StreamToolCallFunction = _StreamToolCallFunction()
+
+
+class _StreamDelta(BaseModel):
+    content: str | None = None
+    tool_calls: list[_StreamToolCall] | None = None
+
+
+class _StreamChoice(BaseModel):
+    delta: _StreamDelta = _StreamDelta()
+
+
+class _StreamChunk(BaseModel):
+    choices: list[_StreamChoice] = []
+
+
+def _streamed_tool_call(events: list[str]) -> tuple[str, str]:
+    """Reassemble the tool call streamed across chunks: the name arrives once and the
+    arguments arrive as fragments, so concatenating both and parsing the arguments as
+    JSON catches a stream that never completes the call or splits its argument JSON."""
+    chunks = [_StreamChunk.model_validate_json(event) for event in events]
+    calls = [call for chunk in chunks for choice in chunk.choices for call in (choice.delta.tool_calls or [])]
+    name = "".join(call.function.name or "" for call in calls)
+    arguments = "".join(call.function.arguments or "" for call in calls)
+    return name, arguments
+
+
+CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
+OPENAI_VISION_BACKEND = "openai/gpt-4o"
+
+# OpenAI caches a shared prompt prefix once it exceeds ~1024 tokens; this is well
+# past that, so a repeat call reports cached prompt tokens.
+CACHE_PREFIX = (
+    "You are a meticulous assistant. Follow these standing instructions exactly. "
+    * 300
+)
+
+
+def _vision_messages() -> list[ChatMessage]:
+    return [
+        ChatMessage(
+            role="user",
+            content=[
+                TextContentPart(text="What animal is in this image? Answer in one word."),
+                ImageContentPart(image_url=ImageUrl(url=CAT_IMAGE_URL)),
+            ],
+        )
+    ]
+
+
+def _assert_describes_cat(response: ChatResponse) -> None:
+    assert response.choices, f"vision returned no choices: {response}"
+    message = response.choices[0].message
+    content = (message.content if message else None) or ""
+    assert "cat" in content.lower() or "feline" in content.lower(), (
+        f"vision response did not describe the image: {content[:200]}"
+    )
+
+
+def _streamed_text(events: list[str]) -> str:
+    """Concatenate the delta content across streamed chunks. Parsing every event as
+    JSON also fails loudly on a truncated or garbled chunk (the vertex/gemini image
+    streaming regression class), so an incomplete stream cannot pass as content."""
+    chunks = [_StreamChunk.model_validate_json(event) for event in events]
+    return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
+
+
+def _assert_streamed_completion(result: StreamingResponse) -> None:
+    """A streamed /chat/completions must deliver real content, not a clean-but-empty
+    stream (the #28991 class on the streaming path)."""
+    assert result.ok and result.is_streaming, f"stream was not established: {result}"
+    assert result.stream_error is None, f"stream carried an error event: {result.stream_error}"
+    assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}"
+    assert _streamed_text(result.stream_events).strip(), (
+        f"stream completed with no content deltas: {result.stream_events[:3]}"
+    )
+
+
+def _bedrock_params() -> LiteLLMParamsBody:
+    return LiteLLMParamsBody(
+        model=BEDROCK_CONVERSE_BACKEND,
+        aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+        aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+        aws_region_name="os.environ/AWS_REGION",
+    )
+
+
+class _WeatherArgs(BaseModel):
+    location: str
+
+
+_WEATHER_TOOL = ChatTool(
+    function=ChatToolFunction(
+        name="get_weather",
+        description="Get the current weather for a location",
+        parameters={
+            "type": "object",
+            "properties": {"location": {"type": "string"}},
+            "required": ["location"],
+        },
+    )
+)
+
+
+def _assert_weather_tool_call(response: ChatResponse) -> None:
+    """The model, forced to call the tool, must return a get_weather call whose
+    arguments parse as JSON and carry a location. A regression that drops tool_calls
+    or emits malformed argument JSON fails here rather than passing on a 200."""
+    assert response.choices, f"chat returned no choices: {response}"
+    message = response.choices[0].message
+    calls = message.tool_calls if message else None
+    assert calls, f"model returned no tool call for a tool-forced prompt: {response}"
+    weather = next((call for call in calls if call.function.name == "get_weather"), None)
+    assert weather is not None, f"expected a get_weather call, got {[c.function.name for c in calls]}"
+    assert weather.function.arguments, f"get_weather call carried no arguments: {weather}"
+    args = _WeatherArgs.model_validate_json(weather.function.arguments)
+    assert args.location.strip(), f"get_weather arguments missing location: {weather.function.arguments}"
+
+
+class _Person(BaseModel):
+    name: str
+    age: int
+
+
+_PERSON_SCHEMA: dict[str, object] = {
+    "type": "json_schema",
+    "json_schema": {
+        "name": "person",
+        "strict": True,
+        "schema": {
+            "type": "object",
+            "properties": {"name": {"type": "string"}, "age": {"type": "integer"}},
+            "required": ["name", "age"],
+            "additionalProperties": False,
+        },
+    },
+}
 
 CHAT_MODELS: tuple[tuple[str, str], ...] = (
     ("gpt-5.5", "openai"),
@@ -219,3 +378,391 @@ class TestHostedVllmChat:
         assert response.choices, f"hosted_vllm chat returned no choices: {response}"
         content = response.choices[0].message.content if response.choices[0].message else None
         assert content and content.strip(), f"hosted_vllm empty content: {response}"
+
+
+class TestOpenAIChatCompletions:
+    """OpenAI /chat/completions, the SDK path the customer runs against the proxy.
+
+    The streamed call must deliver real content deltas (a clean-but-empty stream is
+    the regression), and a non-streamed call must be costed so per-request spend and
+    the response-cost header stay accurate.
+    """
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.basic.stream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_streams_real_content(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-chat-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key,
+            ChatBody(
+                model=model,
+                messages=[
+                    ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}")
+                ],
+                max_tokens=64,
+                stream=True,
+            ),
+        )
+        _assert_streamed_completion(result)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.basic.nonstream.cost_logged",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_logs_cost(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-cost-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")],
+                    max_tokens=16,
+                ),
+            )
+        )
+        assert response.choices, f"openai chat returned no choices: {response}"
+
+        rows = client.proxy.poll_logs_for_key(
+            key, min_rows=1, predicate=lambda rs: any((r.spend or 0) > 0 for r in rs)
+        )
+        priced = [r for r in rows if (r.spend or 0) > 0]
+        assert priced, f"openai chat was not costed on key ...{key[-6:]}: {rows}"
+        assert priced[0].status == "success", f"openai chat spend status={priced[0].status!r}"
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.tool_use.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_returns_tool_call(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-tool-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[
+                        ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.")
+                    ],
+                    tools=[_WEATHER_TOOL],
+                    tool_choice="required",
+                    max_tokens=128,
+                ),
+            )
+        )
+        _assert_weather_tool_call(response)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.structured_output.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_structured_output_conforms_to_schema(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-schema-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content="Extract the person. John Doe is 42 years old.")],
+                    response_format=_PERSON_SCHEMA,
+                    max_tokens=128,
+                ),
+            )
+        )
+        assert response.choices, f"structured output returned no choices: {response}"
+        content = response.choices[0].message.content if response.choices[0].message else None
+        assert content, f"structured output returned empty content: {response}"
+        person = _Person.model_validate_json(content)
+        assert person.name.strip() and person.age == 42, (
+            f"schema-constrained extraction was wrong: {person}"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.thinking.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_reasoning_reports_reasoning_tokens(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-reasoning-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[
+                        ChatMessage(
+                            role="user",
+                            content="A train travels 60 miles in 1.5 hours. What is its average speed in mph?",
+                        )
+                    ],
+                    reasoning_effort="low",
+                    max_tokens=2048,
+                ),
+            )
+        )
+        assert response.choices, f"reasoning call returned no choices: {response}"
+        message = response.choices[0].message
+        assert message and message.content and message.content.strip(), f"reasoning call had no answer: {response}"
+        details = response.usage.completion_tokens_details if response.usage else None
+        assert details and details.reasoning_tokens and details.reasoning_tokens > 0, (
+            f"a reasoning model must report reasoning tokens, got usage={response.usage}"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.vision.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_vision_describes_image(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-vision-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_VISION_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
+        _assert_describes_cat(response)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.prompt_cache_5m.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_prompt_cache_hits_on_repeat(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-cache-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        body = ChatBody(
+            model=model,
+            messages=[
+                ChatMessage(role="system", content=CACHE_PREFIX),
+                ChatMessage(role="user", content="Reply with the single word pong."),
+            ],
+            max_tokens=16,
+        )
+        unwrap(client.proxy.chat(key, body))
+        second = unwrap(client.proxy.chat(key, body))
+
+        details = second.usage.prompt_tokens_details if second.usage else None
+        assert details and details.cached_tokens and details.cached_tokens > 0, (
+            f"a repeated large-prefix prompt must report cached prompt tokens, got usage={second.usage}"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.openai.tool_use.stream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_openai_chat_streams_tool_call(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        require_env("OPENAI_API_KEY")
+        model = f"e2e-openai-tool-stream-{unique_marker()}"
+        model_id = client.proxy.create_model(
+            model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")
+        )
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key,
+            ChatBody(
+                model=model,
+                messages=[
+                    ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.")
+                ],
+                tools=[_WEATHER_TOOL],
+                tool_choice="required",
+                max_tokens=128,
+                stream=True,
+            ),
+        )
+        assert result.ok and result.is_streaming, f"tool stream was not established: {result}"
+        assert result.stream_error is None, f"tool stream carried an error event: {result.stream_error}"
+        name, arguments = _streamed_tool_call(result.stream_events)
+        assert name == "get_weather", f"streamed tool call named {name!r}: {result.stream_events[:5]}"
+        args = _WeatherArgs.model_validate_json(arguments)
+        assert args.location.strip(), f"streamed tool call arguments missing location: {arguments!r}"
+
+
+class TestBedrockConverseChatCompletions:
+    """Bedrock Converse via /chat/completions, the customer's AWS stack. A non-OpenAI
+    provider must return real content on both the non-streamed and streamed paths.
+    """
+
+    def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"{prefix}-{unique_marker()}"
+        model_id = client.proxy.create_model(model, _bedrock_params())
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        return model
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.basic.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_returns_content(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-chat")
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=f"Reply with the single word pong. {unique_marker()}")],
+                    max_tokens=32,
+                ),
+            )
+        )
+        assert response.choices, f"bedrock converse chat returned no choices: {response}"
+        content = response.choices[0].message.content if response.choices[0].message else None
+        assert content and content.strip(), f"bedrock converse returned empty content: {response}"
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.basic.stream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_streams_real_content(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-stream")
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key,
+            ChatBody(
+                model=model,
+                messages=[
+                    ChatMessage(role="user", content=f"Count from 1 to 5, one number per line. {unique_marker()}")
+                ],
+                max_tokens=64,
+                stream=True,
+            ),
+        )
+        _assert_streamed_completion(result)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.tool_use.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_returns_tool_call(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-tool")
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[
+                        ChatMessage(role="user", content="What is the weather in San Francisco? Use the get_weather tool.")
+                    ],
+                    tools=[_WEATHER_TOOL],
+                    tool_choice="required",
+                    max_tokens=128,
+                ),
+            )
+        )
+        _assert_weather_tool_call(response)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.thinking.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_returns_reasoning(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-thinking")
+        key = resources.key()
+
+        response = unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content="What is 17 times 23? Think it through step by step.")],
+                    thinking=ThinkingParam(type="enabled", budget_tokens=1024),
+                    max_tokens=2048,
+                ),
+            )
+        )
+        assert response.choices, f"bedrock thinking returned no choices: {response}"
+        message = response.choices[0].message
+        assert message and message.content and message.content.strip(), (
+            f"bedrock thinking returned no answer content: {response}"
+        )
+        assert message.reasoning_content and message.reasoning_content.strip(), (
+            "thinking was enabled but no reasoning_content came back on the Bedrock Converse path"
+        )
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.vision.nonstream.works",
+        exercised_on=["chat_completions"],
+    )
+    def test_bedrock_converse_chat_vision_describes_image(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = self._register(client, resources, "e2e-bedrock-vision")
+        key = resources.key()
+
+        response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
+        _assert_describes_cat(response)

tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py

@@ -1,9 +1,9 @@
-"""Live e2e: POST /embeddings returns a real vector.
+"""Live e2e: POST /embeddings returns a real vector across OpenAI, Bedrock, Vertex.
 
-Registers an OpenAI embedding deployment at runtime and asserts a non-empty,
-non-zero vector came back. Migrated from
-litellm-regression-tests/tests/test_inference_endpoints.py; the LIT-3167 guard in
-tests/e2e/embeddings/ covers the Gemini embedding path.
+Each test registers the deployment it needs at runtime (deleted on teardown) and
+asserts a non-empty, non-zero vector came back. The LIT-3167 guard in
+tests/e2e/embeddings/ covers the Gemini embedding path; embeddings cost tracking is
+covered by tests/e2e/quota_management/spend_tracking/.
 """
 
 from __future__ import annotations
@@ -20,6 +20,7 @@ pytestmark = pytest.mark.e2e
 
 
 class TestEmbeddingsEndpoint:
+    @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works")
     def test_embeddings_returns_vector(
         self, endpoints_client: EndpointsClient, resources: ResourceManager
     ) -> None:
@@ -40,3 +41,49 @@ class TestEmbeddingsEndpoint:
         assert any(component != 0.0 for component in parsed.first_vector), (
             f"embedding vector is all zeros: {result.body[:300]}"
         )
+
+    @pytest.mark.covers("llm.embeddings.bedrock.basic.nonstream.works")
+    def test_bedrock_embeddings_returns_vector(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = f"e2e-embeddings-bedrock-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="bedrock/amazon.titan-embed-text-v2:0", aws_region_name="us-west-2"
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.embeddings(key, model, "Say this is a test!")
+        require_successful_call(result)
+        parsed = EmbeddingsResult.model_validate_json(result.body)
+        assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}"
+        assert any(component != 0.0 for component in parsed.first_vector), (
+            f"embedding vector is all zeros: {result.body[:300]}"
+        )
+
+    @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works")
+    def test_vertex_embeddings_returns_vector(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = f"e2e-embeddings-vertex-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="vertex_ai/gemini-embedding-2",
+                vertex_project="os.environ/VERTEXAI_PROJECT",
+                vertex_location="us-central1",
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.embeddings(key, model, "Say this is a test!")
+        require_successful_call(result)
+        parsed = EmbeddingsResult.model_validate_json(result.body)
+        assert parsed.first_vector, f"/embeddings returned no vector: {result.body[:300]}"
+        assert any(component != 0.0 for component in parsed.first_vector), (
+            f"embedding vector is all zeros: {result.body[:300]}"
+        )

tests/e2e/llm_translation/test_image_generation_e2e.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import pytest
 
-from e2e_config import unique_marker
+from e2e_config import require_env, unique_marker
 from e2e_http import require_successful_call
 from endpoints_client import EndpointsClient, ImagesResult
 from lifecycle import ResourceManager
@@ -18,7 +18,17 @@ from models import LiteLLMParamsBody
 pytestmark = pytest.mark.e2e
 
 
+def _assert_image_returned(body: str) -> None:
+    parsed = ImagesResult.model_validate_json(body)
+    assert parsed.data, f"/images/generations returned no data: {body[:300]}"
+    first = parsed.data[0]
+    assert first.b64_json or first.url, (
+        f"generated image has neither b64_json nor url: {body[:300]}"
+    )
+
+
 class TestImageGeneration:
+    @pytest.mark.covers("llm.images_generations.openai.basic.nonstream.works")
     def test_image_generation_returns_image(
         self, endpoints_client: EndpointsClient, resources: ResourceManager
     ) -> None:
@@ -34,9 +44,26 @@ class TestImageGeneration:
 
         result = endpoints_client.images(key, model, "Draw a cute cat")
         require_successful_call(result)
-        parsed = ImagesResult.model_validate_json(result.body)
-        assert parsed.data, f"/images/generations returned no data: {result.body[:300]}"
-        first = parsed.data[0]
-        assert first.b64_json or first.url, (
-            f"generated image has neither b64_json nor url: {result.body[:300]}"
+        _assert_image_returned(result.body)
+
+    @pytest.mark.covers("llm.images_generations.bedrock.basic.nonstream.works", exercised_on=["images_generations"])
+    def test_bedrock_image_generation_returns_image(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-bedrock-image-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="bedrock/amazon.titan-image-generator-v2:0",
+                aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+                aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+                aws_region_name="os.environ/AWS_REGION",
+            ),
         )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.images(key, model, "Draw a cute cat")
+        require_successful_call(result)
+        _assert_image_returned(result.body)

tests/e2e/llm_translation/test_messages_azure_foundry_e2e.py

@@ -0,0 +1,155 @@
+"""Live e2e: POST /v1/messages routed to Azure AI Foundry Anthropic deployments.
+
+Registers `azure_ai/<claude>` deployments at runtime and drives the Messages
+endpoint through the gateway across the behaviors an Anthropic client relies on:
+a basic completion, a streamed completion, and tool use (non-streaming and
+streaming). Auth is the Azure API key (`x-api-key`); the deployment reads
+`AZURE_AI_API_BASE` / `AZURE_AI_API_KEY` from the proxy env, so no secret is
+sent in the request.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import unique_marker
+from e2e_http import StreamingResponse, require_successful_call, unwrap
+from endpoints_client import EndpointsClient
+from lifecycle import ResourceManager
+from models import (
+    AnthropicCustomTool,
+    AnthropicMessagesBody,
+    ChatMessage,
+    JsonSchemaProperty,
+    LiteLLMParamsBody,
+    ToolInputSchema,
+)
+
+pytestmark = pytest.mark.e2e
+
+AZURE_FOUNDRY_MODEL = "azure_ai/claude-haiku-4-5"
+
+WEATHER_TOOL = AnthropicCustomTool(
+    name="get_weather",
+    description="Get the current weather for a city.",
+    input_schema=ToolInputSchema(
+        properties={"city": JsonSchemaProperty(type="string")},
+        required=["city"],
+    ),
+)
+
+
+def _assert_streamed_ok(result: StreamingResponse) -> None:
+    require_successful_call(result)
+    assert result.is_streaming, f"response was not streamed: {result.headers}"
+    assert not result.stream_error, f"stream errored: {result.stream_error}"
+    assert result.stream_events, "stream produced no SSE events"
+    assert any("content_block_delta" in event for event in result.stream_events), (
+        "stream carried no content deltas"
+    )
+    assert any("message_stop" in event for event in result.stream_events), (
+        "stream never reached message_stop"
+    )
+
+
+class TestAzureFoundryMessages:
+    def _register(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> tuple[str, str]:
+        model = f"e2e-azure-foundry-messages-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model=AZURE_FOUNDRY_MODEL,
+                api_base="os.environ/AZURE_AI_API_BASE",
+                api_key="os.environ/AZURE_AI_API_KEY",
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        return model, resources.key(models=[model])
+
+    @pytest.mark.covers("llm.messages.azure_foundry.basic.nonstream.works")
+    def test_basic_nonstream(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model, key = self._register(endpoints_client, resources)
+        response = unwrap(
+            endpoints_client.proxy.messages(
+                key,
+                AnthropicMessagesBody(
+                    model=model,
+                    max_tokens=64,
+                    messages=[ChatMessage(role="user", content="Reply with one word.")],
+                ),
+            )
+        )
+        assert response.content, f"no content blocks in response: {response}"
+        text = "".join(block.text or "" for block in response.content if block.type == "text")
+        assert text.strip(), f"/v1/messages returned no text: {response}"
+
+    @pytest.mark.covers("llm.messages.azure_foundry.basic.stream.works")
+    def test_basic_stream(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model, key = self._register(endpoints_client, resources)
+        result = endpoints_client.proxy.messages_stream(
+            key,
+            AnthropicMessagesBody(
+                model=model,
+                max_tokens=64,
+                stream=True,
+                messages=[ChatMessage(role="user", content="Count from one to three.")],
+            ),
+        )
+        _assert_streamed_ok(result)
+
+    @pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works")
+    def test_tool_use_nonstream(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model, key = self._register(endpoints_client, resources)
+        response = unwrap(
+            endpoints_client.proxy.messages(
+                key,
+                AnthropicMessagesBody(
+                    model=model,
+                    max_tokens=256,
+                    tools=[WEATHER_TOOL],
+                    messages=[
+                        ChatMessage(role="user", content="What is the weather in Paris? Use the tool.")
+                    ],
+                ),
+            )
+        )
+        assert response.content, f"no content blocks in response: {response}"
+        assert any(block.type == "tool_use" for block in response.content), (
+            f"model did not call the tool: {response}"
+        )
+
+    @pytest.mark.covers("llm.messages.azure_foundry.tool_use.stream.works")
+    def test_tool_use_stream(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model, key = self._register(endpoints_client, resources)
+        result = endpoints_client.proxy.messages_stream(
+            key,
+            AnthropicMessagesBody(
+                model=model,
+                max_tokens=256,
+                stream=True,
+                tools=[WEATHER_TOOL],
+                messages=[
+                    ChatMessage(role="user", content="What is the weather in Paris? Use the tool.")
+                ],
+            ),
+        )
+        require_successful_call(result)
+        assert result.is_streaming, f"response was not streamed: {result.headers}"
+        assert not result.stream_error, f"stream errored: {result.stream_error}"
+        assert result.stream_events, "stream produced no SSE events"
+        assert any("tool_use" in event for event in result.stream_events), (
+            "stream carried no tool_use block"
+        )
+        assert any("message_stop" in event for event in result.stream_events), (
+            "stream never reached message_stop"
+        )

tests/e2e/llm_translation/test_messages_e2e.py

@@ -1,7 +1,8 @@
 """Live e2e: POST /v1/messages (Anthropic Messages API) returns a real completion.
 
 Registers an Anthropic deployment at runtime, drives the Messages endpoint through
-the gateway, and asserts an assistant message with text came back. Migrated from
+the gateway, and asserts an assistant message with text came back, both
+non-streaming and streamed. Migrated from
 litellm-regression-tests/tests/test_inference_endpoints.py.
 """
 
@@ -9,31 +10,163 @@ from __future__ import annotations
 
 import pytest
 
-from e2e_config import unique_marker
-from e2e_http import require_successful_call
+from e2e_config import require_env, unique_marker
+from e2e_http import require_successful_call, unwrap
 from endpoints_client import EndpointsClient, MessagesResult
 from lifecycle import ResourceManager
-from models import LiteLLMParamsBody
+from models import (
+    AnthropicCustomTool,
+    AnthropicMessagesBody,
+    ChatMessage,
+    JsonSchemaProperty,
+    LiteLLMParamsBody,
+    SpendLogRow,
+    ToolInputSchema,
+)
 
 pytestmark = pytest.mark.e2e
 
+ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5"
+
+WEATHER_TOOL = AnthropicCustomTool(
+    name="get_weather",
+    description="Get the current weather for a city.",
+    input_schema=ToolInputSchema(
+        properties={"city": JsonSchemaProperty(type="string")},
+        required=["city"],
+    ),
+)
+
+
+def _approx_equal(actual: float, expected: float) -> bool:
+    """Within 1% or 1e-9 absolute - spend math, not exact float identity."""
+    return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2)
+
 
 class TestAnthropicMessages:
-    def test_messages_returns_completion(
+    def _register(
         self, endpoints_client: EndpointsClient, resources: ResourceManager
-    ) -> None:
+    ) -> tuple[str, str]:
         model = f"e2e-messages-{unique_marker()}"
         model_id = endpoints_client.create_model(
             model,
             LiteLLMParamsBody(
-                model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY"
+                model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"
             ),
         )
         resources.defer(lambda: endpoints_client.delete_model(model_id))
-        key = resources.key()
+        return model, resources.key()
+
+    @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.works")
+    def test_messages_returns_completion(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model, key = self._register(endpoints_client, resources)
 
         result = endpoints_client.messages(key, model, "reply with one word")
         require_successful_call(result)
         parsed = MessagesResult.model_validate_json(result.body)
         assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}"
         assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}"
+
+    @pytest.mark.covers("llm.messages.anthropic.basic.nonstream.cost_logged")
+    def test_messages_logs_cost_matching_the_response_header(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("ANTHROPIC_API_KEY")
+        model = f"e2e-messages-cost-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY"
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.messages(key, model, f"reply with one word {unique_marker()}")
+        require_successful_call(result)
+        parsed = MessagesResult.model_validate_json(result.body)
+        assert parsed.role == "assistant" and parsed.text.strip(), (
+            f"/v1/messages returned no assistant text: {result.body[:300]}"
+        )
+
+        # The customer reads per-request cost off the response header (LIT-4076), so
+        # it must be present and positive on /v1/messages, not only /chat/completions.
+        header_cost = result.response_cost
+        assert header_cost is not None and header_cost > 0, (
+            "x-litellm-response-cost header missing or non-positive on /v1/messages; "
+            f"headers={result.headers}"
+        )
+
+        # Correlate the spend row by the unique scoped key, not the Anthropic response
+        # id: on /v1/messages the spend-log request_id is the proxy's own call id, which
+        # need not equal the message body id, so an id-based poll can miss a correctly
+        # logged row and time out. The key is fresh per test, so its only priced row is
+        # this call.
+        def _priced(rows: list[SpendLogRow]) -> bool:
+            return any(r.spend is not None and r.spend > 0 for r in rows)
+
+        rows = endpoints_client.proxy.poll_logs_for_key(key, predicate=_priced)
+        priced = [r for r in rows if r.spend is not None and r.spend > 0]
+        assert priced, (
+            f"no priced /spend/logs row landed for key {key} within the poll window; got {rows}"
+        )
+        row = priced[0]
+        assert (row.prompt_tokens or 0) > 0 and (row.completion_tokens or 0) > 0, (
+            f"messages spend row missing token counts, so the cost is not real usage: {row}"
+        )
+        assert row.spend is not None and _approx_equal(row.spend, header_cost), (
+            f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}; "
+            "the customer bills against the header, so the two must match"
+        )
+
+    @pytest.mark.covers("llm.messages.anthropic.basic.stream.works")
+    def test_messages_streams_completion(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model, key = self._register(endpoints_client, resources)
+
+        result = endpoints_client.proxy.messages_stream(
+            key,
+            AnthropicMessagesBody(
+                model=model,
+                max_tokens=64,
+                stream=True,
+                messages=[ChatMessage(role="user", content="Count from one to three.")],
+            ),
+        )
+        require_successful_call(result)
+        assert result.is_streaming, f"response was not streamed: {result.headers}"
+        assert not result.stream_error, f"stream errored: {result.stream_error}"
+        assert result.stream_events, "stream produced no SSE events"
+        assert any("content_block_delta" in event for event in result.stream_events), (
+            "stream carried no content deltas"
+        )
+        assert any("message_stop" in event for event in result.stream_events), (
+            "stream never reached message_stop"
+        )
+
+    @pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works")
+    def test_messages_tool_use(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model, key = self._register(endpoints_client, resources)
+
+        response = unwrap(
+            endpoints_client.proxy.messages(
+                key,
+                AnthropicMessagesBody(
+                    model=model,
+                    max_tokens=256,
+                    tools=[WEATHER_TOOL],
+                    messages=[
+                        ChatMessage(role="user", content="What is the weather in Paris? Use the tool.")
+                    ],
+                ),
+            )
+        )
+        assert response.content, f"no content blocks in response: {response}"
+        assert any(block.type == "tool_use" for block in response.content), (
+            f"model did not call the tool: {response}"
+        )

tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py

@@ -7,7 +7,7 @@ accepted in place on Claude 4.8+/5 (200) but rejected on Claude 4.7 and older
 ("role 'system' is not supported on this model", 400), and a *leading* system
 entry is rejected on every model ("messages.0: use the top-level 'system'
 parameter"). This mirrors Bedrock Invoke (PRs #32578/#32831/#32882); the same
-model-gated hoist now runs for these two providers (Kraken Tech RCA gap #3).
+model-gated hoist now runs for these two providers (customer RCA gap #3).
 
 Flagged models (``supports_mid_conversation_system`` in the cost map: Claude
 4.8+ and the 5 family) must keep the reminder in ``messages`` so the top-level
@@ -88,9 +88,9 @@ def _system_reminder_turn() -> RichMessage:
 
 
 def _post_messages(client: EndpointsClient, key: str, body: RichMessagesRequest) -> Result[MessagesResult]:
-    return client.gateway.transport.post(
+    return client.proxy.transport.post(
         "/v1/messages",
-        headers=client.gateway.transport.bearer(key),
+        headers=client.proxy.transport.bearer(key),
         json=body,
         response_type=MessagesResult,
     )

tests/e2e/llm_translation/test_moderations_e2e.py

@@ -0,0 +1,65 @@
+"""Live e2e: POST /v1/moderations classifies content against the provider policy.
+
+Registers OpenAI's omni moderation model at runtime and asserts the product
+promise on both sides of the decision: clearly violent text comes back flagged
+with at least one policy category tripped, and benign text comes back not flagged.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import unique_marker
+from e2e_http import unwrap
+from endpoints_client import EndpointsClient
+from lifecycle import ResourceManager
+from models import LiteLLMParamsBody
+
+pytestmark = pytest.mark.e2e
+
+VIOLENT_TEXT = "I am going to find you and kill you, and I will hurt everyone you love."
+BENIGN_TEXT = "I enjoyed the sunny afternoon and a relaxing walk in the park today."
+
+
+def _register_moderation_model(
+    endpoints_client: EndpointsClient, resources: ResourceManager
+) -> str:
+    model = f"e2e-moderation-{unique_marker()}"
+    model_id = endpoints_client.create_model(
+        model,
+        LiteLLMParamsBody(
+            model="openai/omni-moderation-latest", api_key="os.environ/OPENAI_API_KEY"
+        ),
+    )
+    resources.defer(lambda: endpoints_client.delete_model(model_id))
+    return model
+
+
+class TestModerations:
+    @pytest.mark.covers("llm.moderations.openai.basic.nonstream.works")
+    def test_moderations_flags_violent_content(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = _register_moderation_model(endpoints_client, resources)
+        key = resources.key()
+
+        result = unwrap(endpoints_client.moderations(key, model, VIOLENT_TEXT))
+        item = result.first
+        assert item is not None, f"/moderations returned no results: {result}"
+        assert item.flagged, f"violent text was not flagged: {item}"
+        assert item.flagged_categories, (
+            f"flagged result reported no true category: {item}"
+        )
+
+    def test_moderations_passes_benign_content(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = _register_moderation_model(endpoints_client, resources)
+        key = resources.key()
+
+        result = unwrap(endpoints_client.moderations(key, model, BENIGN_TEXT))
+        item = result.first
+        assert item is not None, f"/moderations returned no results: {result}"
+        assert not item.flagged, (
+            f"benign text was flagged as {item.flagged_categories}: {item}"
+        )

tests/e2e/llm_translation/test_passthrough_e2e.py

@@ -15,7 +15,8 @@ import pytest
 
 from e2e_config import unique_marker
 from e2e_http import StreamingResponse, require_successful_call
-from models import SpendLogRow
+from lifecycle import ResourceManager
+from models import KeyGenerateBody, SpendLogRow
 from passthrough_client import (
     AnthropicTool,
     GeminiFunctionDeclaration,
@@ -157,3 +158,25 @@ def test_anthropic_passthrough_tool_call_logs_cost(
 
     row = _fetch_cost_breakdown(client, result)
     assert row.custom_llm_provider == "anthropic"
+
+
+class TestPassthroughModelAllowlist:
+    """A passthrough route must honor the calling key's model allow-list.
+
+    The customer fronts native provider calls through the proxy with custom auth,
+    so a key scoped to one model must not reach a different model just because the
+    request goes through the passthrough route rather than /chat/completions.
+    """
+
+    @pytest.mark.covers("other.auth.passthrough.model_allowlist_enforced")
+    def test_passthrough_denies_model_outside_key_allowlist(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        key = client.proxy.generate_key(KeyGenerateBody(models=["gemini-2.5-flash"]))
+        resources.defer(lambda: client.proxy.delete_key(key))
+
+        result = client.anthropic_message(key, "claude-haiku-4-5", f"say hi {unique_marker()}")
+        assert result.status_code == 403, (
+            "a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
+            f"got {result.status_code}: {result.body[:300]}"
+        )

tests/e2e/llm_translation/test_passthrough_headers_e2e.py

@@ -1,29 +1,32 @@
 """Live e2e: custom pass-through endpoints inject configured headers and honor
 x-pass-* client headers (prefix stripped) on the way to the upstream.
 
-The upstream is a real public echo service (httpbin.org/anything). Creating the
-route via POST /config/pass_through_endpoint, calling it with a virtual key, and
-asserting the echo body is the product path operators use; a mock would not
-prove the proxy actually rewrote the outbound request.
+The upstream is the real Anthropic Messages API rather than an echo service:
+Anthropic doesn't echo request headers back, but it does gate real behavior on
+two of them, which is enough to prove forwarding without a mock. A static
+x-api-key configured on the pass-through endpoint (the caller never supplies
+one) must reach upstream, or every call 401s; an invalid x-pass-anthropic-version
+sent by the caller must reach upstream with the prefix stripped, and Anthropic
+echoes the exact value back in its 400 body, so a unique-per-run marker proves
+this specific request's header - not a stale or cached one - got there.
 """
 
 from __future__ import annotations
 
 import pytest
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, Field
 
 from e2e_config import unique_marker
-from e2e_http import AuthHeaders, NoBody, StreamingResponse, require_successful_call, unwrap
+from e2e_http import AuthHeaders, NoBody, require_successful_call, unwrap
+from endpoints_client import MessagesResult
 from lifecycle import ResourceManager
-from models import KeyGenerateBody
+from models import ChatMessage, KeyGenerateBody
 from passthrough_client import PassthroughClient
 
 pytestmark = pytest.mark.e2e
 
-ECHO_TARGET = "https://httpbin.org/anything"
-STATIC_HEADER_NAME = "x-e2e-static-header"
-PASS_HEADER_STEM = "e2e-client-marker"
-PASS_HEADER_NAME = f"x-pass-{PASS_HEADER_STEM}"
+ANTHROPIC_MESSAGES_TARGET = "https://api.anthropic.com/v1/messages"
+MODEL = "claude-haiku-4-5-20251001"
 
 
 class PassThroughCreateBody(BaseModel):
@@ -48,30 +51,26 @@ class PassThroughDeleteParams(BaseModel):
     endpoint_id: str
 
 
-class EchoCallHeaders(AuthHeaders):
+class AnthropicPassThroughHeaders(AuthHeaders):
     content_type: str = Field(default="application/json", serialization_alias="Content-Type")
-    x_pass_e2e_client_marker: str = Field(serialization_alias="x-pass-e2e-client-marker")
+    x_pass_anthropic_version: str = Field(serialization_alias="x-pass-anthropic-version")
 
 
-class EchoBody(BaseModel):
-    ping: str
+class AnthropicMessagesBody(BaseModel):
+    model: str
+    max_tokens: int = 8
+    messages: list[ChatMessage]
 
 
-class EchoResponse(BaseModel):
-    headers: dict[str, str]
-
-
-def _create_passthrough(
-    client: PassthroughClient, *, path: str, static_value: str
-) -> PassThroughEndpoint:
+def _create_passthrough(client: PassthroughClient, *, path: str) -> PassThroughEndpoint:
     created = unwrap(
         client.proxy.transport.post(
             "/config/pass_through_endpoint",
             headers=client.proxy.transport.master,
             json=PassThroughCreateBody(
                 path=path,
-                target=ECHO_TARGET,
-                headers={STATIC_HEADER_NAME: static_value},
+                target=ANTHROPIC_MESSAGES_TARGET,
+                headers={"x-api-key": "os.environ/ANTHROPIC_API_KEY"},
             ),
             response_type=PassThroughCreateResponse,
         )
@@ -92,12 +91,8 @@ def _delete_passthrough(client: PassthroughClient, endpoint_id: str) -> None:
     )
 
 
-def _echo_headers(resp: StreamingResponse) -> dict[str, str]:
-    try:
-        echo = EchoResponse.model_validate_json(resp.body)
-    except ValidationError as exc:
-        pytest.fail(f"echo upstream did not return a headers map: {exc}; body={resp.body[:300]}")
-    return {k.lower(): v for k, v in echo.headers.items()}
+def _messages_body() -> AnthropicMessagesBody:
+    return AnthropicMessagesBody(model=MODEL, messages=[ChatMessage(role="user", content="Say hi.")])
 
 
 class TestPassthroughHeaders:
@@ -110,10 +105,8 @@ class TestPassthroughHeaders:
     ) -> None:
         marker = unique_marker()
         path = f"/e2e-passthrough-headers-{marker}"
-        static_value = f"static-{marker}"
-        client_value = f"client-{marker}"
 
-        endpoint = _create_passthrough(client, path=path, static_value=static_value)
+        endpoint = _create_passthrough(client, path=path)
         assert endpoint.id is not None
         resources.defer(lambda: _delete_passthrough(client, endpoint.id or ""))
 
@@ -128,23 +121,32 @@ class TestPassthroughHeaders:
 
         result = client.proxy.transport.send(
             path,
-            headers=EchoCallHeaders(
+            headers=AnthropicPassThroughHeaders(
                 authorization=f"Bearer {key}",
-                x_pass_e2e_client_marker=client_value,
+                x_pass_anthropic_version="2023-06-01",
             ),
-            json=EchoBody(ping=marker),
+            json=_messages_body(),
         )
         require_successful_call(result)
+        completion = MessagesResult.model_validate_json(result.body)
+        assert completion.text.strip(), (
+            f"static x-api-key must reach Anthropic for the call to succeed at all; got {result.body[:300]}"
+        )
 
-        upstream = _echo_headers(result)
-        assert upstream.get(STATIC_HEADER_NAME) == static_value, (
-            f"configured pass-through header {STATIC_HEADER_NAME!r} not on upstream "
-            f"request; got {upstream}"
+        invalid_version = f"e2e-passhdr-{unique_marker()}"
+        blocked = client.proxy.transport.send(
+            path,
+            headers=AnthropicPassThroughHeaders(
+                authorization=f"Bearer {key}",
+                x_pass_anthropic_version=invalid_version,
+            ),
+            json=_messages_body(),
         )
-        assert upstream.get(PASS_HEADER_STEM) == client_value, (
-            f"x-pass-* header should strip the prefix and forward as {PASS_HEADER_STEM!r}; "
-            f"got {upstream}"
+        assert blocked.status_code == 400, (
+            f"expected Anthropic to reject the invalid anthropic-version, got "
+            f"{blocked.status_code}: {blocked.body[:300]}"
         )
-        assert PASS_HEADER_NAME not in upstream, (
-            "upstream must not see the x-pass- prefix; proxy should strip it"
+        assert invalid_version in blocked.body, (
+            f"x-pass-anthropic-version must reach upstream with the prefix stripped; "
+            f"marker missing from Anthropic's error body: {blocked.body[:300]}"
         )

tests/e2e/llm_translation/test_rerank_e2e.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import pytest
 
-from e2e_config import unique_marker
+from e2e_config import require_env, unique_marker
 from e2e_http import require_successful_call
 from endpoints_client import EndpointsClient, RerankResult
 from lifecycle import ResourceManager
@@ -23,9 +23,20 @@ DOCUMENTS = [
     "Washington, D.C. is the capital of the United States.",
     "Capital punishment has existed in the United States since before it was a country.",
 ]
+QUERY = "What is the capital of the United States?"
+
+
+def _assert_top_n_scored(body: str) -> None:
+    parsed = RerankResult.model_validate_json(body)
+    assert parsed.results, f"/rerank returned no results: {body[:300]}"
+    assert len(parsed.results) <= 3, f"top_n=3 not honored: {body[:300]}"
+    assert parsed.results[0].relevance_score is not None, (
+        f"top rerank result has no relevance_score: {body[:300]}"
+    )
 
 
 class TestRerank:
+    @pytest.mark.covers("llm.rerank.cohere.basic.nonstream.works")
     def test_rerank_scores_top_n(
         self, endpoints_client: EndpointsClient, resources: ResourceManager
     ) -> None:
@@ -37,13 +48,28 @@ class TestRerank:
         resources.defer(lambda: endpoints_client.delete_model(model_id))
         key = resources.key()
 
-        result = endpoints_client.rerank(
-            key, model, "What is the capital of the United States?", DOCUMENTS, top_n=3
-        )
+        result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3)
         require_successful_call(result)
-        parsed = RerankResult.model_validate_json(result.body)
-        assert parsed.results, f"/rerank returned no results: {result.body[:300]}"
-        assert len(parsed.results) <= 3, f"top_n=3 not honored: {result.body[:300]}"
-        assert parsed.results[0].relevance_score is not None, (
-            f"top rerank result has no relevance_score: {result.body[:300]}"
+        _assert_top_n_scored(result.body)
+
+    @pytest.mark.covers("llm.rerank.bedrock.basic.nonstream.works", exercised_on=["rerank"])
+    def test_bedrock_rerank_scores_top_n(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-bedrock-rerank-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="bedrock/amazon.rerank-v1:0",
+                aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+                aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+                aws_region_name="os.environ/AWS_REGION",
+            ),
         )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3)
+        require_successful_call(result)
+        _assert_top_n_scored(result.body)

tests/e2e/llm_translation/test_responses_e2e.py

@@ -13,7 +13,7 @@ from typing import cast
 import pytest
 from pydantic import BaseModel, ValidationError
 
-from e2e_config import unique_marker
+from e2e_config import require_env, unique_marker
 from e2e_http import require_successful_call
 from endpoints_client import (
     EndpointsClient,
@@ -29,6 +29,26 @@ from models import LiteLLMParamsBody
 
 pytestmark = pytest.mark.e2e
 
+BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+
+WEATHER_TOOL = ResponsesFunctionTool(
+    name="get_weather",
+    description="Get the weather for a location",
+    parameters=FunctionParameters(
+        properties={"location": FunctionParameterProperty(type="string")},
+        required=["location"],
+    ),
+)
+
+
+def _bedrock_params() -> LiteLLMParamsBody:
+    return LiteLLMParamsBody(
+        model=BEDROCK_CONVERSE_BACKEND,
+        aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+        aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+        aws_region_name="os.environ/AWS_REGION",
+    )
+
 
 class WeatherArguments(BaseModel):
     location: str
@@ -190,6 +210,84 @@ class TestResponses:
         parsed = ResponsesResult.model_validate_json(result.body)
         assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}"
 
+    @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works")
+    def test_responses_anthropic_returns_function_call(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        model = f"e2e-responses-{unique_marker()}"
+        model_id = endpoints_client.create_model(
+            model,
+            LiteLLMParamsBody(
+                model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY"
+            ),
+        )
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.responses_with_tools(
+            key,
+            model,
+            "What is the weather in San Francisco? Use the get_weather tool.",
+            [
+                ResponsesFunctionTool(
+                    name="get_weather",
+                    description="Get the weather for a location",
+                    parameters=FunctionParameters(
+                        properties={"location": FunctionParameterProperty(type="string")},
+                        required=["location"],
+                    ),
+                )
+            ],
+        )
+        require_successful_call(result)
+        parsed = ResponsesResult.model_validate_json(result.body)
+        function_call = next(
+            (call for call in parsed.function_calls if call.name == "get_weather"),
+            None,
+        )
+        assert function_call is not None, f"no get_weather function call: {result.body[:500]}"
+        assert function_call.arguments is not None
+        raw_arguments = cast(object, json.loads(function_call.arguments))
+        arguments = WeatherArguments.model_validate(raw_arguments)
+        assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
+
+    @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works")
+    def test_responses_bedrock_returns_completion(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-responses-{unique_marker()}"
+        model_id = endpoints_client.create_model(model, _bedrock_params())
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.responses(key, model, "reply with one word")
+        require_successful_call(result)
+        parsed = ResponsesResult.model_validate_json(result.body)
+        assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}"
+
+    @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works")
+    def test_responses_bedrock_returns_function_call(
+        self, endpoints_client: EndpointsClient, resources: ResourceManager
+    ) -> None:
+        require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
+        model = f"e2e-responses-{unique_marker()}"
+        model_id = endpoints_client.create_model(model, _bedrock_params())
+        resources.defer(lambda: endpoints_client.delete_model(model_id))
+        key = resources.key()
+
+        result = endpoints_client.responses_with_tools(
+            key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL]
+        )
+        require_successful_call(result)
+        parsed = ResponsesResult.model_validate_json(result.body)
+        function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None)
+        assert function_call is not None, f"no get_weather function call over bedrock: {result.body[:500]}"
+        assert function_call.arguments is not None
+        raw_arguments = cast(object, json.loads(function_call.arguments))
+        arguments = WeatherArguments.model_validate(raw_arguments)
+        assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
+
 
 def _parse_stream_event(
     event: str,

tests/e2e/load/conftest.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
+import os
 from collections.abc import Iterator
 
 import pytest
 from requests import RequestException
 
+from e2e_config import WEEKLY_ANOMALY_OPT_IN_ENV
 from e2e_http import NoBody, Success
 from load_client import LoadClient, build_client
 from load_constants import LOAD_MODEL
@@ -18,6 +20,22 @@ LOAD_MODEL_PARAMS = LiteLLMParamsBody(
 )
 
 
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: list[pytest.Item]
+) -> None:
+    if os.environ.get(WEEKLY_ANOMALY_OPT_IN_ENV):
+        return
+    deselected = [
+        item for item in items if item.get_closest_marker("weekly") is not None
+    ]
+    if not deselected:
+        return
+    config.hook.pytest_deselected(items=deselected)
+    items[:] = [
+        item for item in items if item.get_closest_marker("weekly") is None
+    ]
+
+
 @pytest.fixture(scope="session")
 def client(proxy: ProxyClient) -> LoadClient:
     return build_client(proxy)
@@ -33,10 +51,8 @@ def _model_is_servable(proxy: ProxyClient, model_name: str) -> bool:
     return isinstance(result, Success) and any(entry.id == model_name for entry in result.data.data)
 
 
-@pytest.fixture(scope="session", autouse=True)
-def _ensure_load_model(  # pyright: ignore[reportUnusedFunction]  # pytest autouse session fixture, wired by name
-    client: LoadClient,
-) -> Iterator[None]:
+@pytest.fixture(scope="session")
+def ensure_load_model(client: LoadClient) -> Iterator[None]:
     proxy = client.proxy
     if _model_is_servable(proxy, LOAD_MODEL):
         yield
@@ -60,7 +76,9 @@ def _ensure_load_model(  # pyright: ignore[reportUnusedFunction]  # pytest autou
 
 
 @pytest.fixture
-def load_key(resources: ResourceManager, client: LoadClient) -> str:
+def load_key(
+    resources: ResourceManager, client: LoadClient, ensure_load_model: None
+) -> str:
     key = client.proxy.generate_key(KeyGenerateBody(models=[LOAD_MODEL], user_id="e2e-load"))
     resources.defer(lambda: client.proxy.delete_key(key))
     return key

tests/e2e/load/session_anomaly.py

@@ -0,0 +1,299 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+
+from pydantic import BaseModel
+
+from e2e_config import unique_marker
+from e2e_http import Result, Success
+from models import CacheControl, RichMessage, TextBlock
+from transport import Transport
+
+
+class SessionMessagesRequest(BaseModel):
+    model: str
+    max_tokens: int = 128
+    system: list[TextBlock]
+    messages: list[RichMessage]
+
+
+class SessionUsage(BaseModel):
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_creation_input_tokens: int = 0
+    cache_read_input_tokens: int = 0
+
+
+class SessionContentBlock(BaseModel):
+    type: str | None = None
+    text: str | None = None
+
+
+class SessionMessagesResponse(BaseModel):
+    content: list[SessionContentBlock] = []
+    usage: SessionUsage = SessionUsage()
+
+    @property
+    def text(self) -> str:
+        return "".join(block.text or "" for block in self.content)
+
+
+@dataclass(frozen=True, slots=True)
+class TurnMetric:
+    turn_index: int
+    ok: bool
+    latency_seconds: float
+    uncached_input_tokens: int
+    cache_read_tokens: int
+    cache_creation_tokens: int
+    failure: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class AnomalyReport:
+    planned_turns: int
+    attempted_turns: int
+    failed_turns: int
+    warm_turns: int
+    warm_uncached_input_tokens: int
+    warm_cache_read_tokens: int
+    warm_cache_creation_tokens: int
+    p95_turn_seconds: float
+
+    @property
+    def error_ratio(self) -> float:
+        return self.failed_turns / self.planned_turns if self.planned_turns else 1.0
+
+    @property
+    def warm_cache_read_share(self) -> float:
+        billed = (
+            self.warm_uncached_input_tokens
+            + self.warm_cache_read_tokens
+            + self.warm_cache_creation_tokens
+        )
+        return self.warm_cache_read_tokens / billed if billed else 0.0
+
+
+def _system_prefix_block(marker: str) -> TextBlock:
+    text = " ".join(
+        f"Project context paragraph {index} for session {marker}." for index in range(300)
+    )
+    return TextBlock(text=text, cache_control=CacheControl())
+
+
+def _user_turn_text(marker: str, turn_index: int) -> str:
+    notes = " ".join(
+        f"Working note {index} of turn {turn_index} in session {marker}."
+        for index in range(80)
+    )
+    return f"Reply with one short sentence.\n{notes}"
+
+
+def _reminder_turn() -> RichMessage:
+    return RichMessage(
+        role="system",
+        content=[
+            TextBlock(
+                text="<system-reminder>Keep the answer to one short sentence.</system-reminder>"
+            )
+        ],
+    )
+
+
+def _without_cache_control(message: RichMessage) -> RichMessage:
+    return RichMessage(
+        role=message.role,
+        content=[TextBlock(text=block.text) for block in message.content],
+    )
+
+
+RETRY_BACKOFF_SECONDS = 2.0
+
+
+def retried(
+    call: Callable[[], Result[SessionMessagesResponse]],
+    attempts: int,
+    backoff_seconds: float = RETRY_BACKOFF_SECONDS,
+    sleep: Callable[[float], None] = time.sleep,
+) -> Result[SessionMessagesResponse]:
+    result = call()
+    if isinstance(result, Success) or attempts <= 1:
+        return result
+    sleep(backoff_seconds)
+    return retried(call, attempts - 1, backoff_seconds, sleep)
+
+
+def _metric(
+    result: Result[SessionMessagesResponse], turn_index: int, latency_seconds: float
+) -> TurnMetric:
+    if isinstance(result, Success):
+        usage = result.data.usage
+        return TurnMetric(
+            turn_index=turn_index,
+            ok=True,
+            latency_seconds=latency_seconds,
+            uncached_input_tokens=usage.input_tokens,
+            cache_read_tokens=usage.cache_read_input_tokens,
+            cache_creation_tokens=usage.cache_creation_input_tokens,
+            failure=None,
+        )
+    return TurnMetric(
+        turn_index=turn_index,
+        ok=False,
+        latency_seconds=latency_seconds,
+        uncached_input_tokens=0,
+        cache_read_tokens=0,
+        cache_creation_tokens=0,
+        failure=repr(result),
+    )
+
+
+def _drive_turns(
+    transport: Transport,
+    key: str,
+    model: str,
+    marker: str,
+    system_block: TextBlock,
+    history: tuple[RichMessage, ...],
+    turn_index: int,
+    remaining_turns: int,
+    attempts_per_turn: int,
+) -> tuple[TurnMetric, ...]:
+    if remaining_turns == 0:
+        return ()
+    user_turn = RichMessage(
+        role="user",
+        content=[
+            TextBlock(
+                text=_user_turn_text(marker, turn_index), cache_control=CacheControl()
+            )
+        ],
+    )
+    started = time.monotonic()
+    result = retried(
+        lambda: transport.post(
+            "/v1/messages",
+            headers=transport.bearer(key),
+            json=SessionMessagesRequest(
+                model=model,
+                system=[system_block],
+                messages=[*history, user_turn],
+            ),
+            response_type=SessionMessagesResponse,
+        ),
+        attempts_per_turn,
+    )
+    turn = _metric(result, turn_index, time.monotonic() - started)
+    if not isinstance(result, Success):
+        return (turn,)
+    assistant_turn = RichMessage(
+        role="assistant", content=[TextBlock(text=result.data.text or "Understood.")]
+    )
+    return (
+        turn,
+        *_drive_turns(
+            transport,
+            key,
+            model,
+            marker,
+            system_block,
+            (
+                *history,
+                _without_cache_control(user_turn),
+                _reminder_turn(),
+                assistant_turn,
+            ),
+            turn_index + 1,
+            remaining_turns - 1,
+            attempts_per_turn,
+        ),
+    )
+
+
+def run_session(
+    transport: Transport, key: str, model: str, turns: int, attempts_per_turn: int
+) -> tuple[TurnMetric, ...]:
+    marker = unique_marker()
+    return _drive_turns(
+        transport,
+        key,
+        model,
+        marker,
+        _system_prefix_block(marker),
+        (),
+        1,
+        turns,
+        attempts_per_turn,
+    )
+
+
+def run_concurrent_sessions(
+    transport: Transport,
+    key: str,
+    model: str,
+    sessions: int,
+    turns_per_session: int,
+    attempts_per_turn: int,
+) -> tuple[TurnMetric, ...]:
+    with ThreadPoolExecutor(max_workers=sessions) as pool:
+        futures = [
+            pool.submit(
+                run_session, transport, key, model, turns_per_session, attempts_per_turn
+            )
+            for _ in range(sessions)
+        ]
+        return tuple(turn for future in futures for turn in future.result())
+
+
+def settled_spend(
+    read_spend: Callable[[], float],
+    poll_interval: float,
+    settle_seconds: float,
+    timeout_seconds: float,
+    now: Callable[[], float] = time.monotonic,
+    sleep: Callable[[float], None] = time.sleep,
+) -> float:
+    deadline = now() + timeout_seconds + settle_seconds
+
+    def settle(previous: float, stable_since: float) -> float:
+        current = read_spend()
+        observed = now()
+        since = stable_since if current == previous else observed
+        if current > 0 and observed - since >= settle_seconds:
+            return current
+        if observed >= deadline:
+            raise AssertionError(
+                f"key spend never held a stable non-zero value for {settle_seconds}s "
+                f"within {timeout_seconds + settle_seconds}s (last read {current}); "
+                f"spend stopped being recorded, which is itself a spend anomaly"
+            )
+        sleep(poll_interval)
+        return settle(current, since)
+
+    return settle(-1.0, now())
+
+
+def _p95(latencies: tuple[float, ...]) -> float:
+    if not latencies:
+        return 0.0
+    ranked = sorted(latencies)
+    return ranked[max(0, -(-len(ranked) * 95 // 100) - 1)]
+
+
+def summarize(turns: tuple[TurnMetric, ...], planned_turns: int) -> AnomalyReport:
+    warm = tuple(turn for turn in turns if turn.ok and turn.turn_index >= 2)
+    return AnomalyReport(
+        planned_turns=planned_turns,
+        attempted_turns=len(turns),
+        failed_turns=planned_turns - sum(1 for turn in turns if turn.ok),
+        warm_turns=len(warm),
+        warm_uncached_input_tokens=sum(turn.uncached_input_tokens for turn in warm),
+        warm_cache_read_tokens=sum(turn.cache_read_tokens for turn in warm),
+        warm_cache_creation_tokens=sum(turn.cache_creation_tokens for turn in warm),
+        p95_turn_seconds=_p95(
+            tuple(turn.latency_seconds for turn in turns if turn.ok)
+        ),
+    )

tests/e2e/load/test_session_anomaly.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from itertools import count, repeat
+
+import pytest
+
+from e2e_http import NetworkError, Success
+from session_anomaly import (
+    SessionMessagesResponse,
+    TurnMetric,
+    retried,
+    settled_spend,
+    summarize,
+)
+
+
+def _ok_turn(turn_index: int) -> TurnMetric:
+    return TurnMetric(
+        turn_index=turn_index,
+        ok=True,
+        latency_seconds=1.0,
+        uncached_input_tokens=10,
+        cache_read_tokens=100,
+        cache_creation_tokens=5,
+        failure=None,
+    )
+
+
+def _failed_turn(turn_index: int) -> TurnMetric:
+    return TurnMetric(
+        turn_index=turn_index,
+        ok=False,
+        latency_seconds=1.0,
+        uncached_input_tokens=0,
+        cache_read_tokens=0,
+        cache_creation_tokens=0,
+        failure="NetworkError()",
+    )
+
+
+class TestSummarizePlannedTurns:
+    def test_session_aborted_on_first_turn_counts_all_its_planned_turns_as_failed(
+        self,
+    ) -> None:
+        completed_session = tuple(_ok_turn(index) for index in range(1, 7))
+        aborted_session = (_failed_turn(1),)
+
+        report = summarize((*completed_session, *aborted_session), planned_turns=12)
+
+        assert report.attempted_turns == 7
+        assert report.failed_turns == 6
+        assert report.error_ratio == 0.5
+
+    def test_all_planned_turns_completing_reports_zero_failures(self) -> None:
+        report = summarize(
+            tuple(_ok_turn(index) for index in range(1, 7)), planned_turns=6
+        )
+
+        assert report.failed_turns == 0
+        assert report.error_ratio == 0.0
+
+
+class TestRetried:
+    def test_transient_failures_then_success_returns_the_success(self) -> None:
+        outcome = Success(data=SessionMessagesResponse())
+        calls = iter(
+            (NetworkError(message="overloaded"), NetworkError(message="overloaded"), outcome)
+        )
+
+        result = retried(lambda: next(calls), attempts=3, sleep=lambda _: None)
+
+        assert result is outcome
+
+    def test_exhausted_attempts_return_the_last_failure(self) -> None:
+        last_attempt = NetworkError(message="still overloaded")
+        never_reached = NetworkError(message="a fourth attempt would break the budget")
+        calls = iter(
+            (NetworkError(message="overloaded"), last_attempt, never_reached)
+        )
+
+        result = retried(lambda: next(calls), attempts=2, sleep=lambda _: None)
+
+        assert result is last_attempt
+        assert next(calls) is never_reached
+
+    def test_first_try_success_never_sleeps(self) -> None:
+        def sleep_means_retry(_: float) -> None:
+            raise AssertionError("slept after a successful attempt")
+
+        result = retried(
+            lambda: Success(data=SessionMessagesResponse()),
+            attempts=3,
+            sleep=sleep_means_retry,
+        )
+
+        assert isinstance(result, Success)
+
+
+class TestSettledSpend:
+    def test_partial_total_between_batch_flushes_is_not_accepted_as_final(self) -> None:
+        reads = iter((0.1, 0.1, 0.1, 0.35, 0.35, 0.35, 0.35, 0.35))
+        ticks = count(0.0, 2.5)
+
+        spend = settled_spend(
+            lambda: next(reads),
+            poll_interval=5.0,
+            settle_seconds=10.0,
+            timeout_seconds=100.0,
+            now=lambda: next(ticks),
+            sleep=lambda _: None,
+        )
+
+        assert spend == 0.35
+
+    def test_spend_that_never_stabilizes_raises(self) -> None:
+        reads = (0.1 * step for step in count(1))
+        ticks = count(0.0, 2.5)
+
+        with pytest.raises(AssertionError, match="spend anomaly"):
+            settled_spend(
+                lambda: next(reads),
+                poll_interval=5.0,
+                settle_seconds=5.0,
+                timeout_seconds=10.0,
+                now=lambda: next(ticks),
+                sleep=lambda _: None,
+            )
+
+    def test_spend_that_never_becomes_nonzero_raises(self) -> None:
+        reads = repeat(0.0)
+        ticks = count(0.0, 2.5)
+
+        with pytest.raises(AssertionError, match="spend anomaly"):
+            settled_spend(
+                lambda: next(reads),
+                poll_interval=5.0,
+                settle_seconds=5.0,
+                timeout_seconds=10.0,
+                now=lambda: next(ticks),
+                sleep=lambda _: None,
+            )

tests/e2e/load/test_weekly_session_anomaly_e2e.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+
+from e2e_config import (
+    ANOMALY_MAX_ERROR_RATIO,
+    ANOMALY_MAX_KEY_SPEND_USD,
+    ANOMALY_MAX_P95_TURN_SECONDS,
+    ANOMALY_MIN_WARM_CACHE_READ_SHARE,
+    ANOMALY_SESSIONS,
+    ANOMALY_SPEND_SETTLE_SECONDS,
+    ANOMALY_TURN_ATTEMPTS,
+    ANOMALY_TURNS_PER_SESSION,
+    unique_marker,
+)
+from lifecycle import ResourceManager
+from load_client import LoadClient
+from models import KeyGenerateBody, LiteLLMParamsBody
+from proxy_client import ProxyClient
+from session_anomaly import run_concurrent_sessions, settled_spend, summarize
+
+pytestmark = [pytest.mark.e2e, pytest.mark.load, pytest.mark.weekly]
+
+
+@dataclass(frozen=True, slots=True)
+class AnomalyRoute:
+    route_id: str
+    params: LiteLLMParamsBody
+
+
+ANOMALY_ROUTES = (
+    AnomalyRoute(
+        route_id="anthropic",
+        params=LiteLLMParamsBody(model="anthropic/claude-sonnet-5"),
+    ),
+    AnomalyRoute(
+        route_id="bedrock_invoke",
+        params=LiteLLMParamsBody(
+            model="bedrock/invoke/us.anthropic.claude-sonnet-5",
+            aws_region_name="us-east-1",
+        ),
+    ),
+)
+
+
+def _route_id(route: AnomalyRoute) -> str:
+    return route.route_id
+
+
+def _settled_key_spend(proxy: ProxyClient, key: str) -> float:
+    return settled_spend(
+        lambda: proxy.key_info(key).spend or 0.0,
+        proxy.poll_interval,
+        ANOMALY_SPEND_SETTLE_SECONDS,
+        proxy.poll_timeout,
+    )
+
+
+class TestWeeklySessionAnomaly:
+    @pytest.mark.covers("reliability.perf.session_anomaly.under_slo")
+    @pytest.mark.parametrize("route", ANOMALY_ROUTES, ids=_route_id)
+    def test_session_load_stays_within_baselines(
+        self, client: LoadClient, resources: ResourceManager, route: AnomalyRoute
+    ) -> None:
+        model_name = f"weekly-anomaly-{route.route_id}-{unique_marker()}"
+        model_id = client.proxy.create_model(model_name, route.params)
+        resources.defer(lambda: client.proxy.delete_model(model_id))
+        key = client.proxy.generate_key(
+            KeyGenerateBody(models=[model_name], key_alias=model_name)
+        )
+        resources.defer(lambda: client.proxy.delete_key(key))
+
+        turns = run_concurrent_sessions(
+            client.proxy.transport,
+            key,
+            model_name,
+            ANOMALY_SESSIONS,
+            ANOMALY_TURNS_PER_SESSION,
+            ANOMALY_TURN_ATTEMPTS,
+        )
+        report = summarize(turns, ANOMALY_SESSIONS * ANOMALY_TURNS_PER_SESSION)
+        failures = tuple(turn.failure for turn in turns if turn.failure)
+        print(f"{route.route_id} anomaly report: {report}")
+
+        assert report.error_ratio <= ANOMALY_MAX_ERROR_RATIO, (
+            f"{route.route_id}: {report.failed_turns}/{report.planned_turns} planned "
+            f"turns failed or never ran because their session aborted "
+            f"({report.error_ratio:.1%} > {ANOMALY_MAX_ERROR_RATIO:.1%} allowed); "
+            f"error rate is anomalously high. Failures: {failures}"
+        )
+        assert report.warm_turns > 0, (
+            f"{route.route_id}: no session got past its first turn, so cache and "
+            f"latency baselines have nothing to read. Failures: {failures}"
+        )
+        assert report.warm_cache_read_share >= ANOMALY_MIN_WARM_CACHE_READ_SHARE, (
+            f"{route.route_id}: warm turns read only {report.warm_cache_read_share:.1%} "
+            f"of billed input tokens from the prompt cache "
+            f"(read={report.warm_cache_read_tokens}, "
+            f"creation={report.warm_cache_creation_tokens}, "
+            f"uncached={report.warm_uncached_input_tokens}), below the "
+            f"{ANOMALY_MIN_WARM_CACHE_READ_SHARE:.0%} floor; the cached prefix is "
+            f"being invalidated between turns (the mid-conversation-system cache "
+            f"collapse signature) or caching stopped working"
+        )
+        assert report.warm_cache_creation_tokens > 0, (
+            f"{route.route_id}: warm turns wrote 0 cache-creation tokens across "
+            f"{report.warm_turns} turns; the moving cache breakpoint stopped writing "
+            f"new prefix increments"
+        )
+        assert report.p95_turn_seconds <= ANOMALY_MAX_P95_TURN_SECONDS, (
+            f"{route.route_id}: p95 turn time {report.p95_turn_seconds:.1f}s exceeds "
+            f"the {ANOMALY_MAX_P95_TURN_SECONDS:.0f}s ceiling under "
+            f"{ANOMALY_SESSIONS} concurrent sessions; turn times are anomalously slow"
+        )
+
+        spend = _settled_key_spend(client.proxy, key)
+        assert spend <= ANOMALY_MAX_KEY_SPEND_USD, (
+            f"{route.route_id}: gateway recorded ${spend:.4f} for "
+            f"{report.attempted_turns} turns, above the "
+            f"${ANOMALY_MAX_KEY_SPEND_USD} ceiling; spend per session is "
+            f"anomalously high (cache regressions surface here as 2-3x spend)"
+        )

tests/e2e/load/weekly_anomaly_config.yml

@@ -0,0 +1,3 @@
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  store_model_in_db: true

tests/e2e/management/management_client.py

@@ -14,6 +14,10 @@ from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, Un
 from models import (
     ChatBody,
     ChatMessage,
+    CustomerDeleteBody,
+    CustomerInfoParams,
+    CustomerNewBody,
+    CustomerResponse,
     KeyBlockBody,
     KeyDeleteBody,
     KeyGenerateBody,
@@ -270,6 +274,35 @@ class ManagementClient:
             )
         ).user_id
 
+    def create_customer(self, user_id: str) -> str:
+        _ = unwrap(
+            self.proxy.transport.post(
+                "/customer/new",
+                headers=self.proxy.transport.master,
+                json=CustomerNewBody(user_id=user_id),
+                response_type=CustomerResponse,
+            )
+        )
+        return user_id
+
+    def customer_info(self, end_user_id: str) -> CustomerResponse:
+        return unwrap(
+            self.proxy.transport.get(
+                "/customer/info",
+                headers=self.proxy.transport.master,
+                params=CustomerInfoParams(end_user_id=end_user_id),
+                response_type=CustomerResponse,
+            )
+        )
+
+    def delete_customer(self, user_id: str) -> None:
+        _ = self.proxy.transport.post(
+            "/customer/delete",
+            headers=self.proxy.transport.master,
+            json=CustomerDeleteBody(user_ids=[user_id]),
+            response_type=NoBody,
+        )
+
     def update_user(self, body: UserUpdateBody) -> None:
         _ = unwrap(
             self.proxy.transport.post(