merge: sync with main, resolve model_prices_and_context_window_backup.json conflict

.circleci/config.yml

@@ -69,9 +69,11 @@ jobs:
       - run:
           name: Install Python
           command: |
-            choco install python --version=3.11.0 -y
+            choco install python --version=3.11.0 -y --no-progress --force
             refreshenv
             python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
       - run:
           name: Install Dependencies
           command: |

.github/workflows/publish_enterprise.yml

@@ -19,6 +19,7 @@ jobs:
     if: github.repository == 'BerriAI/litellm'
     permissions:
       contents: write
+      pull-requests: write
     defaults:
       run:
         working-directory: enterprise
@@ -56,14 +57,33 @@ jobs:
       - name: Build
         run: poetry build
 
-      - name: Commit version bump
+      - name: Commit version bump and create PR
+        id: create-pr
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           cd ..
+          BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
+          git checkout -b "$BRANCH"
           git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
           git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
-          git push
+          git push origin "$BRANCH" --force
+          gh pr create \
+            --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
+            --body "Version bump for litellm-enterprise. Merge to update main." \
+            --head "$BRANCH" \
+            --base main \
+          || true
+          PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
+          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        run: |
+          gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Publish to PyPI
         env:

Dockerfile

@@ -49,7 +49,7 @@ USER root
 
 # Install runtime dependencies (libsndfile needed for audio processing on ARM64)
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
     # levels inside its dependency tree. `npm install -g <pkg>` only creates a
     # SEPARATE global package, it does NOT replace npm's internal copies.
@@ -70,7 +70,15 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    npm cache clean --force
+    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
+    # actual installed versions instead of the stale declared dependencies.
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
+    # no longer visible to image scanners.  The globally installed npm@latest
+    # at /usr/local/lib/node_modules/npm/ remains fully functional.
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -96,6 +104,7 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \

docker/Dockerfile.custom_ui

@@ -19,7 +19,7 @@ RUN apt-get update && apt-get upgrade -y \
         libgnutls30 \
         libc6 && \
     apt-get install -y nodejs npm && \
-    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -36,7 +36,10 @@ RUN apt-get update && apt-get upgrade -y \
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    npm cache clean --force
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    apt-get purge -y npm
 
 # Copy the UI source into the container
 COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

docker/Dockerfile.database

@@ -50,7 +50,7 @@ USER root
 
 # Install runtime dependencies
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -67,7 +67,10 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    npm cache clean --force
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -85,6 +88,7 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \

docker/Dockerfile.dev

@@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
     nodejs \
     npm \
     && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
+    && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
     && GLOBAL="$(npm root -g)" \
     && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -92,7 +92,10 @@ RUN apt-get update && apt-get upgrade -y \
     && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
        done \
-    && npm cache clean --force
+    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+    && npm cache clean --force \
+    && apt-get purge -y npm
 
 WORKDIR /app
 
@@ -114,6 +117,7 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \

docker/Dockerfile.non_root

@@ -106,7 +106,7 @@ RUN for i in 1 2 3; do \
     apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
     done \
   && apk upgrade --no-cache nodejs \
-  && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
+  && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
   && GLOBAL="$(npm root -g)" \
   && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -123,7 +123,10 @@ RUN for i in 1 2 3; do \
   && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
      done \
-  && npm cache clean --force
+  && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+  && npm cache clean --force \
+  && { apk del --no-cache npm 2>/dev/null || true; }
 
 # Copy artifacts from builder
 COPY --from=builder /app/requirements.txt /app/requirements.txt
@@ -169,6 +172,7 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \

docs/my-website/docs/a2a.md

@@ -20,6 +20,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
 | Logging | ✅ |
 | Load Balancing | ✅ |
 | Streaming | ✅ |
+| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
 
 
 :::tip

docs/my-website/docs/a2a_iteration_budgets.md

@@ -0,0 +1,188 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Agent Iteration Budgets
+
+Control runaway costs from agentic loops with per-session iteration and budget caps.
+
+## Overview
+
+When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
+
+| Control | Description |
+|---------|-------------|
+| **Max Iterations** | Hard cap on the number of LLM calls per session |
+| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
+
+Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
+
+## Trace-ID Enforcement
+
+LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
+
+| Flag | Description |
+|------|-------------|
+| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
+| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
+
+## Configuring via UI
+
+When creating an agent in the LiteLLM Admin UI:
+
+1. Navigate to the **Agents** tab and click **Add Agent**
+2. In the **Agent Settings** step, expand the **Tracing** section
+3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
+4. Set **Max Iterations** to cap the number of LLM calls per session
+5. Set **Max Budget Per Session ($)** to cap spend per session
+
+The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
+
+## Configuring via API
+
+Set trace-id enforcement on the agent itself:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_to_agent": true,
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+```
+
+Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true,
+      "max_iterations": 25,
+      "max_budget_per_session": 5.00
+    }
+  }'
+```
+
+## How It Works
+
+### Session Tracking
+
+Callers identify their session by including a `session_id` in one of these ways:
+- **Header**: `x-litellm-trace-id: my-session-123`
+- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
+
+### Max Iterations
+
+When `max_iterations` is set in agent `litellm_params`:
+- Each LLM call for a session increments a counter
+- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
+- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
+
+### Max Budget Per Session
+
+When `max_budget_per_session` is set in agent `litellm_params`:
+- After each successful LLM call, the response cost is accumulated for the session
+- Before each call, the accumulated spend is checked against the budget
+- When spend exceeds the budget, the request receives a **429 Too Many Requests**
+- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
+
+## Example
+
+Create an agent with max 25 iterations and a $5 budget cap:
+
+<Tabs>
+<TabItem value="ui" label="Via UI">
+
+1. Go to **Agents** → **Add Agent**
+2. Configure your agent (name, model, etc.)
+3. In **Agent Settings**, expand the **Tracing** section
+4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
+5. Set **Max Iterations** to `25`
+6. Set **Max Budget Per Session** to `5.00`
+7. Proceed to create a new key for the agent
+8. Click **Create Agent**
+
+</TabItem>
+<TabItem value="api" label="Via API">
+
+```bash
+# 1. Create the agent with trace-id enforcement
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+
+# 2. Create a key for the agent
+curl -X POST 'http://localhost:4000/key/generate' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_id": "<agent_id_from_step_1>",
+    "key_alias": "my-research-agent-key"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Making Calls with Session Tracking
+
+```bash
+curl -X POST 'http://localhost:4000/chat/completions' \
+  -H 'Authorization: Bearer sk-agent-key-xxx' \
+  -H 'x-litellm-trace-id: session-abc-123' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+After 25 calls or $5 spent within this session, subsequent requests will receive:
+
+```json
+{
+  "error": {
+    "message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
+    "type": "budget_exceeded",
+    "code": 429
+  }
+}
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
+| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

docs/my-website/docs/proxy/config_settings.md

@@ -355,7 +355,7 @@ router_settings:
 | set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging) If true, sets the logging level to verbose. |
 | retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. |
 | provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) |
-| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
+| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. **Required** for `model_info.max_input_tokens` enforcement. Default: false. [More information here](reliability) |
 | model_group_retry_policy | Dict[str, RetryPolicy] | [SDK-only arg] Set retry policy for model groups. |
 | context_window_fallbacks | List[Dict[str, List[str]]] | Fallback models for context window violations. |
 | redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
@@ -804,6 +804,7 @@ router_settings:
 | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
 | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
 | LITELLM_MASTER_KEY | Master key for proxy authentication
+| LITELLM_MAX_BUDGET_PER_SESSION_TTL | TTL in seconds for session budget counters used by the max-budget-per-session limiter. Default is 3600 (1 hour)
 | LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
 | LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit)
 | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)

docs/my-website/docs/proxy/dynamic_rate_limit.md

@@ -3,6 +3,8 @@
 
 Prevent projects from gobbling too much tpm/rpm.
 
+**See Also:** [Request Prioritization](../scheduler.md) - Prioritize LLM API requests in high-traffic by adding them to a priority queue.
+
 Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
 
 ## Quick Start Usage

docs/my-website/docs/proxy/reliability.md

@@ -713,6 +713,34 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 
 [**See Code**](https://github.com/BerriAI/litellm/blob/c9e6b05cfb20dfb17272218e2555d6b496c47f6f/litellm/router.py#L2163)
 
+:::important
+**`enable_pre_call_checks` is required** for context-window enforcement. Without it, requests are sent to the provider regardless of input token count. Set `enable_pre_call_checks: true` in `router_settings` in your config.
+:::
+
+#### Custom max_input_tokens per deployment
+
+You can override the default context limit for a deployment by setting `max_input_tokens` in `model_info`. This is useful for testing, rate-limiting long prompts, or enforcing stricter limits than the provider's default.
+
+**Both** of the following are required:
+
+1. **`router_settings.enable_pre_call_checks: true`** — enables pre-call checks
+2. **`model_info.max_input_tokens`** on the deployment — overrides the limit for that model
+
+```yaml
+router_settings:
+  enable_pre_call_checks: true  # Required for enforcement
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      max_input_tokens: 10  # Override: reject prompts > 10 tokens
+```
+
+If a request exceeds the limit, LiteLLM raises `ContextWindowExceededError` with details like `Model=gpt-4o, Max Input Tokens=10, Got=306`.
+
 **1. Setup config**
 
 For azure deployments, set the base model. Pick the base model from [this list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json), all the azure models start with azure/.

docs/my-website/docs/proxy/users.md

@@ -10,6 +10,8 @@ import TabItem from '@theme/TabItem';
 
 **Team member budgets**: Set individual spending limits within the team's shared budget
 
+**Agent budgets**: Set rate limits (tpm/rpm) and session-level caps (iterations, dollar budget) on agents [**Jump**](#agents)
+
 ***If a key belongs to a team, the team budget is applied, not the user's personal budget.***
 :::
 
@@ -420,6 +422,109 @@ Expected response on failure
 </Tabs>
 
 
+### Agents
+
+Set budgets and rate limits on agents registered with LiteLLM's [Agent Gateway](../a2a.md). You can control:
+- **Per-agent rate limits**: `tpm_limit` and `rpm_limit` on the agent itself
+- **Per-session rate limits**: `session_tpm_limit` and `session_rpm_limit` applied per session
+- **Per-session iteration cap**: `max_iterations` in agent `litellm_params`
+- **Per-session budget cap**: `max_budget_per_session` in agent `litellm_params`
+
+<Tabs>
+<TabItem value="agent-rate-limits" label="Agent Rate Limits">
+
+Set `tpm_limit` and `rpm_limit` on the agent to cap total throughput across all sessions.
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "tpm_limit": 100000,
+    "rpm_limit": 100
+  }'
+```
+
+</TabItem>
+<TabItem value="session-rate-limits" label="Session Rate Limits">
+
+Set `session_tpm_limit` and `session_rpm_limit` to cap throughput per individual session.
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "session_tpm_limit": 50000,
+    "session_rpm_limit": 50
+  }'
+```
+
+</TabItem>
+<TabItem value="session-budgets" label="Session Budgets">
+
+Set `max_iterations` and `max_budget_per_session` in agent `litellm_params` to cap individual sessions. Requires `require_trace_id_on_calls_by_agent` so LiteLLM can track calls per session.
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true,
+      "max_iterations": 25,
+      "max_budget_per_session": 5.00
+    }
+  }'
+```
+
+When a session exceeds the limit, requests receive a **429 Too Many Requests** response.
+
+See the [Agent Iteration Budgets](../a2a_iteration_budgets) guide for full details.
+
+</TabItem>
+</Tabs>
+
+:::info
+
+You can also update rate limits on existing agents using `PATCH /v1/agents/{agent_id}`:
+
+```bash
+curl -X PATCH 'http://localhost:4000/v1/agents/<agent_id>' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "tpm_limit": 200000,
+    "rpm_limit": 200,
+    "session_tpm_limit": 50000,
+    "session_rpm_limit": 50
+  }'
+```
+
+:::
+
+
 ### Customers
 
 Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user**
@@ -685,6 +790,31 @@ These headers indicate:
 - 1 request remaining for the GPT-4 model for key=`sk-ulGNRXWtv7M0lFnnsQk0wQ`
 - 179 tokens remaining for the GPT-4 model for key=`sk-ulGNRXWtv7M0lFnnsQk0wQ`
 
+</TabItem>
+<TabItem value="per-agent" label="Per Agent">
+
+Set rate limits on agents registered with the [Agent Gateway](../a2a.md).
+
+**Agent-level limits** cap total throughput across all sessions:
+
+```shell
+curl -X POST 'http://0.0.0.0:4000/v1/agents' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{"agent_name": "my-agent", "agent_card_params": {"name": "my-agent", "description": "My agent", "url": "http://my-agent:8080", "version": "1.0.0"}, "tpm_limit": 100000, "rpm_limit": 100}'
+```
+
+**Session-level limits** cap throughput per individual session:
+
+```shell
+curl -X POST 'http://0.0.0.0:4000/v1/agents' \
+--header 'Authorization: Bearer sk-1234' \
+--header 'Content-Type: application/json' \
+--data '{"agent_name": "my-agent", "agent_card_params": {"name": "my-agent", "description": "My agent", "url": "http://my-agent:8080", "version": "1.0.0"}, "session_tpm_limit": 50000, "session_rpm_limit": 50}'
+```
+
+You can also set **max_iterations** (call count cap) and **max_budget_per_session** (dollar cap) per session via `litellm_params`. See [Agent Iteration Budgets](../a2a_iteration_budgets) for details.
+
 </TabItem>
 <TabItem value="per-end-user" label="For customers">
 

docs/my-website/docs/search/index.md

@@ -2,7 +2,7 @@
 
 | Feature | Supported | 
 |---------|-----------|
-| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup`, `duckduckgo`, `searchapi` |
+| Supported Providers | `perplexity`, `tavily`, `parallel_ai`, `exa_ai`, `brave`, `google_pse`, `dataforseo`, `firecrawl`, `searxng`, `linkup`, `duckduckgo`, `searchapi`, `serper` |
 | Cost Tracking | ✅ |
 | Logging | ✅ |
 | Load Balancing | ❌ |
@@ -210,7 +210,7 @@ See the [official Perplexity Search documentation](https://docs.perplexity.ai/ap
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `query` | string or array | Yes | Search query. Can be a single string or array of strings |
-| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, `"linkup"`, `"duckduckgo"`, or `"searchapi"` |
+| `search_provider` | string | Yes (SDK) | The search provider to use: `"perplexity"`, `"tavily"`, `"parallel_ai"`, `"exa_ai"`, `"brave"`, `"google_pse"`, `"dataforseo"`, `"firecrawl"`, `"searxng"`, `"linkup"`, `"duckduckgo"`, `"searchapi"`, or `"serper"` |
 | `search_tool_name` | string | Yes (Proxy) | Name of the search tool configured in `config.yaml` |
 | `max_results` | integer | No | Maximum number of results to return (1-20). Default: 10 |
 | `search_domain_filter` | array | No | List of domains to filter results (max 20 domains) |
@@ -276,6 +276,7 @@ The response follows Perplexity's search format with the following structure:
 | Firecrawl | `FIRECRAWL_API_KEY` | `firecrawl` |
 | SearXNG | `SEARXNG_API_BASE` (required) | `searxng` |
 | Linkup | `LINKUP_API_KEY` | `linkup` |
+| Serper | `SERPER_API_KEY` | `serper` |
 | DuckDuckGo | `DUCKDUCKGO_API_BASE` | `duckduckgo` |
 | SearchAPI.io | `SEARCHAPI_API_KEY` | `searchapi` |
 

docs/my-website/docs/search/serper.md

@@ -0,0 +1,77 @@
+# Serper Search
+
+**Get API Key:** [https://serper.dev](https://serper.dev)
+
+## LiteLLM Python SDK
+
+```python showLineNumbers title="Serper Search"
+import os
+from litellm import search
+
+os.environ["SERPER_API_KEY"] = "your-api-key"
+
+response = search(
+    query="latest AI developments",
+    search_provider="serper",
+    max_results=5
+)
+```
+
+## LiteLLM AI Gateway
+
+### 1. Setup config.yaml
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gpt-5
+    litellm_params:
+      model: gpt-5
+      api_key: os.environ/OPENAI_API_KEY
+
+search_tools:
+  - search_tool_name: serper-search
+    litellm_params:
+      search_provider: serper
+      api_key: os.environ/SERPER_API_KEY
+```
+
+### 2. Start the proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+### 3. Test the search endpoint
+
+```bash showLineNumbers title="Test Request"
+curl http://0.0.0.0:4000/v1/search/serper-search \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "query": "latest AI developments",
+    "max_results": 5
+  }'
+```
+
+## Provider-specific Parameters
+
+```python showLineNumbers title="Serper Search with Provider-specific Parameters"
+import os
+from litellm import search
+
+os.environ["SERPER_API_KEY"] = "your-api-key"
+
+response = search(
+    query="latest tech news",
+    search_provider="serper",
+    max_results=10,
+    # Serper-specific parameters
+    gl="us",                    # Country/geolocation code
+    hl="en",                    # Language code
+    autocorrect=False,          # Disable autocorrect
+    tbs="qdr:d",               # Time filter: past day ('qdr:h' hour, 'qdr:w' week, 'qdr:m' month)
+    page=2                      # Page number
+)
+```

docs/my-website/package.json

@@ -61,7 +61,7 @@
     "mermaid": ">=11.10.0",
     "gray-matter": "4.0.3",
     "glob": ">=11.1.0",
-    "tar": ">=7.5.8",
+    "tar": ">=7.5.10",
     "minimatch": ">=10.2.4",
     "diff": ">=8.0.3",
     "@isaacs/brace-expansion": ">=5.0.1",

docs/my-website/sidebars.js

@@ -542,7 +542,8 @@ const sidebars = {
             "a2a_invoking_agents",
             "a2a_agent_headers",
             "a2a_cost_tracking",
-            "a2a_agent_permissions"
+            "a2a_agent_permissions",
+            "a2a_iteration_budgets"
           ],
         },
         "assistants",
@@ -683,6 +684,7 @@ const sidebars = {
             "search/firecrawl",
             "search/searxng",
             "search/linkup",
+            "search/serper",
           ]
         },
         "skills",

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-enterprise"
-version = "0.1.33"
+version = "0.1.34"
 description = "Package for LiteLLM Enterprise features"
 authors = ["BerriAI"]
 readme = "README.md"

litellm-js/spend-logs/package.json

@@ -12,8 +12,8 @@
   },
   "overrides": {
     "glob": ">=11.1.0",
-    "tar": ">=7.5.8",
-    "minimatch": ">=10.2.1",
+    "tar": ">=7.5.10",
+    "minimatch": ">=10.2.4",
     "diff": ">=8.0.3",
     "@isaacs/brace-expansion": ">=5.0.1",
     "@babel/traverse": ">=7.23.2",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260304175016_add_spend_to_agent_table/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN     "spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+

litellm-proxy-extras/litellm_proxy_extras/migrations/20260305000000_add_rate_limits_to_agents/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "tpm_limit" INTEGER;
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "rpm_limit" INTEGER;
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "session_tpm_limit" INTEGER;
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN "session_rpm_limit" INTEGER;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -68,6 +68,11 @@ model LiteLLM_AgentsTable {
   agent_access_groups String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
+  spend      Float    @default(0.0)
+  tpm_limit  Int?
+  rpm_limit  Int?
+  session_tpm_limit Int?
+  session_rpm_limit Int?
   created_at    DateTime               @default(now()) @map("created_at")
   created_by String
   updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-proxy-extras"
-version = "0.4.52"
+version = "0.4.53"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 authors = ["BerriAI"]
 readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "0.4.52"
+version = "0.4.53"
 version_files = [
     "pyproject.toml:version",
     "../requirements.txt:litellm-proxy-extras==",

litellm/batches/batch_utils.py

@@ -198,9 +198,8 @@ async def _get_batch_output_file_content_as_dictionary(
                        Required for Azure and other providers that need authentication
     """
     from litellm.files.main import afile_content
-    from litellm.proxy.openai_files_endpoints.common_utils import (
-        _is_base64_encoded_unified_file_id,
-    )
+    from litellm.proxy.openai_files_endpoints.common_utils import \
+        _is_base64_encoded_unified_file_id
 
     if custom_llm_provider == "vertex_ai":
         raise ValueError("Vertex AI does not support file content retrieval")
@@ -227,7 +226,7 @@ async def _get_batch_output_file_content_as_dictionary(
     credentials = _extract_file_access_credentials(litellm_params)
     file_content_kwargs.update(credentials)
     
-    _file_content = await afile_content(**file_content_kwargs)
+    _file_content = await afile_content(**file_content_kwargs)  # type: ignore[reportArgumentType]
     return _get_file_content_as_dictionary(_file_content.content)
 
 

litellm/fine_tuning/main.py

@@ -126,6 +126,18 @@ async def acreate_fine_tuning_job(
         raise e
 
 
+def _build_fine_tuning_job_data(model, training_file, hyperparameters, suffix, validation_file, integrations, seed):
+    return FineTuningJobCreate(
+        model=model,
+        training_file=training_file,
+        hyperparameters=hyperparameters,
+        suffix=suffix,
+        validation_file=validation_file,
+        integrations=integrations,
+        seed=seed,
+    )
+
+
 def _resolve_fine_tuning_timeout(
     timeout: Any,
     custom_llm_provider: str,
@@ -206,19 +218,9 @@ def create_fine_tuning_job(
                 or os.getenv("OPENAI_API_KEY")
             )
 
-            create_fine_tuning_job_data = FineTuningJobCreate(
-                model=model,
-                training_file=training_file,
-                hyperparameters=_oai_hyperparameters,
-                suffix=suffix,
-                validation_file=validation_file,
-                integrations=integrations,
-                seed=seed,
-            )
-
-            create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump(
-                exclude_none=True
-            )
+            create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
+                model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
+            ).model_dump(exclude_none=True)
 
             response = openai_fine_tuning_apis_instance.create_fine_tuning_job(
                 api_base=api_base,
@@ -260,20 +262,10 @@ def create_fine_tuning_job(
             # Prepare Azure-specific parameters for extra_body
             extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
             
-            create_fine_tuning_job_data = FineTuningJobCreate(
-                model=model,
-                training_file=training_file,
-                hyperparameters=_oai_hyperparameters,
-                suffix=suffix,
-                validation_file=validation_file,
-                integrations=integrations,
-                seed=seed,
-            )
+            create_fine_tuning_job_data_dict = _build_fine_tuning_job_data(
+                model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
+            ).model_dump(exclude_none=True)
 
-            create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump(
-                exclude_none=True
-            )
-            
             # Add extra_body if it has Azure-specific parameters
             if extra_body:
                 create_fine_tuning_job_data_dict["extra_body"] = extra_body
@@ -303,18 +295,11 @@ def create_fine_tuning_job(
             vertex_credentials = optional_params.vertex_credentials or get_secret_str(
                 "VERTEXAI_CREDENTIALS"
             )
-            create_fine_tuning_job_data = FineTuningJobCreate(
-                model=model,
-                training_file=training_file,
-                hyperparameters=_oai_hyperparameters,
-                suffix=suffix,
-                validation_file=validation_file,
-                integrations=integrations,
-                seed=seed,
-            )
             response = vertex_fine_tuning_apis_instance.create_fine_tuning_job(
                 _is_async=_is_async,
-                create_fine_tuning_job_data=create_fine_tuning_job_data,
+                create_fine_tuning_job_data=_build_fine_tuning_job_data(
+                    model, training_file, _oai_hyperparameters, suffix, validation_file, integrations, seed,
+                ),
                 vertex_credentials=vertex_credentials,
                 vertex_project=vertex_ai_project,
                 vertex_location=vertex_ai_location,

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import time
 import traceback
-from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
+from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union, cast
 
 import litellm
 from litellm._logging import verbose_logger
@@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.types.llms.databricks import DatabricksTool
 from litellm.types.llms.openai import (
     ChatCompletionThinkingBlock,
+    ImageURLListItem,
     OpenAIModerationResponse,
 )
 from litellm.types.utils import (
@@ -26,13 +27,13 @@ from litellm.types.utils import (
     Function,
     HiddenParams,
     ImageResponse,
-    PromptTokensDetailsWrapper,
 )
 from litellm.types.utils import Logprobs as TextCompletionLogprobs
 from litellm.types.utils import (
     Message,
     ModelResponse,
     ModelResponseStream,
+    PromptTokensDetailsWrapper,
     RerankResponse,
     StreamingChoices,
     TextChoices,
@@ -52,6 +53,24 @@ _MODEL_RESPONSE_FIELDS: frozenset = frozenset(ModelResponse.model_fields.keys())
 }
 
 
+def _normalize_images_for_message(
+    images: Optional[List[dict]],
+) -> Optional[List[ImageURLListItem]]:
+    """
+    Ensure each image has an 'index' field, as required by ImageURLListItem.
+    Some providers (e.g. OpenRouter) return images without index.
+    """
+    if not images:
+        return cast(Optional[List[ImageURLListItem]], images)
+    normalized: List[ImageURLListItem] = []
+    for i, img in enumerate(images):
+        if isinstance(img, dict) and "index" not in img:
+            normalized.append(cast(ImageURLListItem, {**img, "index": i}))
+        else:
+            normalized.append(cast(ImageURLListItem, img))
+    return normalized
+
+
 def _safe_convert_created_field(created_value) -> int:
     """
     Safely convert a 'created' field value to an integer.
@@ -591,7 +610,9 @@ def convert_to_model_response_object(  # noqa: PLR0915
                         reasoning_content=reasoning_content,
                         thinking_blocks=thinking_blocks,
                         annotations=choice["message"].get("annotations", None),
-                        images=choice["message"].get("images", None),
+                        images=_normalize_images_for_message(
+                            choice["message"].get("images", None)
+                        ),
                     )
                     finish_reason = choice.get("finish_reason", None)
                 if finish_reason is None:

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -476,13 +476,15 @@ class ChunkProcessor:
             "prompt_tokens_details": prompt_tokens_details,
         }
 
-    def count_reasoning_tokens(self, response: ModelResponse) -> int:
-        reasoning_tokens = 0
+    def count_reasoning_tokens(self, response: ModelResponse) -> Optional[int]:
+        reasoning_tokens: Optional[int] = None
         for choice in response.choices:
             if (
                 hasattr(cast(Choices, choice).message, "reasoning_content")
                 and cast(Choices, choice).message.reasoning_content is not None
             ):
+                if reasoning_tokens is None:
+                    reasoning_tokens = 0
                 reasoning_tokens += token_counter(
                     text=cast(Choices, choice).message.reasoning_content,
                     count_response_tokens=True,

litellm/llms/anthropic/skills/transformation.py

@@ -77,8 +77,8 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig):
             api_base = AnthropicModelInfo.get_api_base()
 
         if skill_id:
-            return f"{api_base}/v1/skills/{skill_id}?beta=true"
-        return f"{api_base}/v1/{endpoint}?beta=true"
+            return f"{api_base}/v1/skills/{skill_id}"
+        return f"{api_base}/v1/{endpoint}"
 
     def transform_create_skill_request(
         self,

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -334,24 +334,67 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         """
         Parse direct JSON response (non-streaming).
 
-        JSON response structure:
-        {
-            "result": {
-                "role": "assistant",
-                "content": [{"text": "..."}]
-            }
-        }
+        Supports multiple agent response schemas:
+        1. {"result": {"role": "assistant", "content": [{"text": "..."}]}} - standard AgentCore
+        2. {"response": [{"text": "..."}]} - Strands agent format
+        3. {"result": "plain text"} or {"response": "plain text"} - simple string
+        4. Fallback: raw JSON as content string
         """
-        result = response_json.get("result", {})
+        # Guard: if json.loads() returned a non-dict (e.g. array or primitive),
+        # skip strategy matching and fall back to raw JSON string
+        if not isinstance(response_json, dict):
+            verbose_logger.warning(
+                "AgentCore: JSON response is not a dict. "
+                "Returning raw JSON as content."
+            )
+            return AgentCoreParsedResponse(
+                content=json.dumps(response_json),
+                usage=None,
+                final_message=None,
+            )
 
-        # Extract content using the same helper as SSE parsing
-        content = self._extract_content_from_message(result)  # type: ignore
+        # Strategy 1: {"result": {"content": [{"text": "..."}]}} - standard AgentCore format
+        if "result" in response_json and isinstance(response_json["result"], dict):
+            result = response_json["result"]
+            content = self._extract_content_from_message(result)  # type: ignore
+            return AgentCoreParsedResponse(
+                content=content,
+                usage=None,
+                final_message=result,  # type: ignore
+            )
 
-        # JSON responses don't include usage data
+        # Strategy 2: {"response": [{"text": "..."}]} - Strands agent content blocks
+        if "response" in response_json and isinstance(
+            response_json["response"], list
+        ):
+            content = self._extract_content_from_message(
+                {"content": response_json["response"]}  # type: ignore
+            )
+            return AgentCoreParsedResponse(
+                content=content,
+                usage=None,
+                final_message=None,
+            )
+
+        # Strategy 3: string values - {"result": "text"} or {"response": "text"}
+        for key in ("result", "response"):
+            val = response_json.get(key)
+            if isinstance(val, str):
+                return AgentCoreParsedResponse(
+                    content=val,
+                    usage=None,
+                    final_message=None,
+                )
+
+        # Strategy 4: fallback - return raw JSON as content
+        verbose_logger.warning(
+            f"AgentCore: Could not extract content from JSON response keys "
+            f"{list(response_json.keys())}. Returning raw JSON as content."
+        )
         return AgentCoreParsedResponse(
-            content=content,
+            content=json.dumps(response_json),
             usage=None,
-            final_message=result,  # type: ignore
+            final_message=None,
         )
 
     def _get_parsed_response(
@@ -589,7 +632,64 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             additional_args={"complete_input_dict": data},
         )
 
-        # Wrap the generator in CustomStreamWrapper
+        # Check if response is JSON (agent used sync return) instead of SSE
+        content_type = response.headers.get("content-type", "").lower()
+        if "application/json" in content_type:
+            verbose_logger.debug(
+                "AgentCore streaming: received JSON response instead of SSE, "
+                "converting to single-chunk stream"
+            )
+            try:
+                body = response.read()
+                response_json = json.loads(body)
+            except (json.JSONDecodeError, Exception) as e:
+                raise BedrockError(
+                    status_code=response.status_code,
+                    message=f"AgentCore: Failed to read/parse JSON response body: {e}",
+                )
+            parsed = self._parse_json_response(response_json)
+
+            def _json_as_sync_stream():
+                # Content chunk
+                content_chunk = ModelResponseStream(
+                    id=f"chatcmpl-{uuid.uuid4()}",
+                    created=0,
+                    model=model,
+                    object="chat.completion.chunk",
+                )
+                content_chunk.choices = [
+                    StreamingChoices(
+                        finish_reason=None,
+                        index=0,
+                        delta=Delta(content=parsed["content"], role="assistant"),
+                    )
+                ]
+                yield content_chunk
+
+                # Stop sentinel chunk (matches SSE path convention)
+                stop_chunk = ModelResponseStream(
+                    id=f"chatcmpl-{uuid.uuid4()}",
+                    created=0,
+                    model=model,
+                    object="chat.completion.chunk",
+                )
+                stop_chunk.choices = [
+                    StreamingChoices(
+                        finish_reason="stop",
+                        index=0,
+                        delta=Delta(),
+                    )
+                ]
+                yield stop_chunk
+
+            return CustomStreamWrapper(
+                completion_stream=_json_as_sync_stream(),
+                model=model,
+                custom_llm_provider="bedrock",
+                logging_obj=logging_obj,
+            )
+
+        # SSE stream (text/event-stream or default) - use existing SSE parser
         return CustomStreamWrapper(
             completion_stream=self._stream_agentcore_response_sync(response, model),
             model=model,
@@ -746,7 +846,64 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             additional_args={"complete_input_dict": data},
         )
 
-        # Wrap the async generator in CustomStreamWrapper
+        # Check if response is JSON (agent used sync return) instead of SSE
+        content_type = response.headers.get("content-type", "").lower()
+        if "application/json" in content_type:
+            verbose_logger.debug(
+                "AgentCore streaming: received JSON response instead of SSE, "
+                "converting to single-chunk stream"
+            )
+            try:
+                body = await response.aread()
+                response_json = json.loads(body)
+            except (json.JSONDecodeError, Exception) as e:
+                raise BedrockError(
+                    status_code=response.status_code,
+                    message=f"AgentCore: Failed to read/parse JSON response body: {e}",
+                )
+            parsed = self._parse_json_response(response_json)
+
+            async def _json_as_async_stream() -> AsyncGenerator[ModelResponseStream, None]:
+                # Content chunk
+                content_chunk = ModelResponseStream(
+                    id=f"chatcmpl-{uuid.uuid4()}",
+                    created=0,
+                    model=model,
+                    object="chat.completion.chunk",
+                )
+                content_chunk.choices = [
+                    StreamingChoices(
+                        finish_reason=None,
+                        index=0,
+                        delta=Delta(content=parsed["content"], role="assistant"),
+                    )
+                ]
+                yield content_chunk
+
+                # Stop sentinel chunk (matches SSE path convention)
+                stop_chunk = ModelResponseStream(
+                    id=f"chatcmpl-{uuid.uuid4()}",
+                    created=0,
+                    model=model,
+                    object="chat.completion.chunk",
+                )
+                stop_chunk.choices = [
+                    StreamingChoices(
+                        finish_reason="stop",
+                        index=0,
+                        delta=Delta(),
+                    )
+                ]
+                yield stop_chunk
+
+            return CustomStreamWrapper(
+                completion_stream=_json_as_async_stream(),
+                model=model,
+                custom_llm_provider="bedrock",
+                logging_obj=logging_obj,
+            )
+
+        # SSE stream (text/event-stream or default) - use existing SSE parser
         return CustomStreamWrapper(
             completion_stream=self._stream_agentcore_response(response, model),
             model=model,

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py

@@ -108,6 +108,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         _anthropic_request.pop("stream", None)
         # Bedrock Invoke doesn't support output_format parameter
         _anthropic_request.pop("output_format", None)
+        # Bedrock Invoke doesn't support output_config parameter
+        # Fixes: https://github.com/BerriAI/litellm/issues/22797
+        _anthropic_request.pop("output_config", None)
         if "anthropic_version" not in _anthropic_request:
             _anthropic_request["anthropic_version"] = self.anthropic_version
 

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -419,6 +419,10 @@ class AmazonAnthropicClaudeMessagesConfig(
                 anthropic_messages_request=anthropic_messages_request,
             )
 
+        # 5b. Strip `output_config` — Bedrock Invoke doesn't support it
+        # Fixes: https://github.com/BerriAI/litellm/issues/22797
+        anthropic_messages_request.pop("output_config", None)
+
         # 5a. Remove `custom` field from tools (Bedrock doesn't support it)
         # Claude Code sends `custom: {defer_loading: true}` on tool definitions,
         # which causes Bedrock to reject the request with "Extra inputs are not permitted"

litellm/llms/custom_httpx/llm_http_handler.py

@@ -152,6 +152,59 @@ else:
     LiteLLMLoggingObj = Any
 
 
+def _sanitize_anthropic_messages_empty_text_blocks(
+    messages: List[Dict],
+) -> List[Dict]:
+    """
+    Strip empty text content blocks from Anthropic-format messages.
+
+    Claude's API returns assistant messages with ``{"type": "text", "text": ""}``
+    alongside ``tool_use`` blocks, but rejects them when sent back in subsequent
+    requests.  This helper removes those empty text blocks so the /v1/messages
+    native path doesn't forward them as-is.
+
+    - If a content list contains a mix of empty text blocks and other blocks
+      (e.g. tool_use), the empty text blocks are removed.
+    - If *all* blocks in a content list are empty text, the content is replaced
+      with a single non-empty placeholder to avoid sending an empty array.
+
+    Ref: https://github.com/BerriAI/litellm/issues/22930
+    """
+    sanitized: List[Dict] = []
+    for message in messages:
+        content = message.get("content")
+        if not isinstance(content, list):
+            sanitized.append(message)
+            continue
+
+        filtered = [
+            block
+            for block in content
+            if not (
+                isinstance(block, dict)
+                and block.get("type") == "text"
+                and not block.get("text", "").strip()
+            )
+        ]
+
+        if filtered == content:
+            # Nothing was removed — keep original message as-is.
+            sanitized.append(message)
+        elif filtered:
+            # Some empty text blocks removed, but other content remains.
+            new_message = message.copy()
+            new_message["content"] = filtered
+            sanitized.append(new_message)
+        else:
+            # All blocks were empty text blocks. Replace with a placeholder
+            # so we don't send an empty content array.
+            new_message = message.copy()
+            new_message["content"] = [{"type": "text", "text": "..."}]
+            sanitized.append(new_message)
+
+    return sanitized
+
+
 class BaseLLMHTTPHandler:
     async def _make_common_async_call(
         self,
@@ -1905,6 +1958,13 @@ class BaseLLMHTTPHandler:
                     anthropic_messages_optional_request_params, path
                 )
 
+        # Sanitize empty text content blocks from messages before forwarding.
+        # Claude's API returns assistant messages with empty text blocks
+        # ({"type": "text", "text": ""}) alongside tool_use blocks, but rejects
+        # them when sent back. Strip these to prevent 400 errors.
+        # Ref: https://github.com/BerriAI/litellm/issues/22930
+        messages = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
         # Prepare request body
         request_body = anthropic_messages_provider_config.transform_anthropic_messages_request(
             model=model,

litellm/llms/serper/search/__init__.py

@@ -0,0 +1,6 @@
+"""
+Serper Search API module.
+"""
+from litellm.llms.serper.search.transformation import SerperSearchConfig
+
+__all__ = ["SerperSearchConfig"]

litellm/llms/serper/search/transformation.py

@@ -0,0 +1,167 @@
+"""
+Calls Serper's /search endpoint to search Google.
+
+Serper API Reference: https://serper.dev
+"""
+from typing import Dict, List, Optional, TypedDict, Union
+
+import httpx
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.search.transformation import (
+    BaseSearchConfig,
+    SearchResponse,
+    SearchResult,
+)
+from litellm.secret_managers.main import get_secret_str
+
+
+class _SerperSearchRequestRequired(TypedDict):
+    """Required fields for Serper Search API request."""
+    q: str  # Required - search query
+
+
+class SerperSearchRequest(_SerperSearchRequestRequired, total=False):
+    """
+    Serper Search API request format.
+    Based on: https://serper.dev
+    """
+    num: int  # Optional - number of results to return, default 10
+    page: int  # Optional - page number (default 1)
+    gl: str  # Optional - country/geolocation code (e.g., "us", "gb")
+    hl: str  # Optional - language code (e.g., "en", "de")
+    location: str  # Optional - specific location for search targeting
+    autocorrect: bool  # Optional - enable autocorrect (default True)
+    tbs: str  # Optional - time-based search filter (e.g., "qdr:h", "qdr:d", "qdr:w")
+
+
+class SerperSearchConfig(BaseSearchConfig):
+    SERPER_API_BASE = "https://google.serper.dev"
+    
+    @staticmethod
+    def ui_friendly_name() -> str:
+        return "Serper"
+    
+    def validate_environment(
+        self,
+        headers: Dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+        **kwargs,
+    ) -> Dict:
+        """
+        Validate environment and return headers.
+        """
+        api_key = api_key or get_secret_str("SERPER_API_KEY")
+        if not api_key:
+            raise ValueError("SERPER_API_KEY is not set. Set `SERPER_API_KEY` environment variable.")
+        headers["X-API-KEY"] = api_key
+        headers["Content-Type"] = "application/json"
+        return headers
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        optional_params: dict,
+        data: Optional[Union[Dict, List[Dict]]] = None,
+        **kwargs,
+    ) -> str:
+        """
+        Get complete URL for Search endpoint.
+        """
+        api_base = api_base or get_secret_str("SERPER_API_BASE") or self.SERPER_API_BASE
+        api_base = api_base.rstrip("/")
+        
+        if not api_base.endswith("/search"):
+            api_base = f"{api_base}/search"
+
+        return api_base
+
+    def transform_search_request(
+        self,
+        query: Union[str, List[str]],
+        optional_params: dict,
+        **kwargs,
+    ) -> Dict:
+        """
+        Transform Search request to Serper API format.
+        
+        Args:
+            query: Search query (string or list of strings). Serper only supports single string queries.
+            optional_params: Optional parameters for the request
+                - max_results: Maximum number of search results -> maps to `num`
+                - search_domain_filter: List of domains -> appended as site: clauses to `q`
+                - country: Country code filter (e.g., 'US', 'GB') -> maps to `gl` (lowercased)
+            
+        Returns:
+            Dict with typed request data following SerperSearchRequest spec
+        """
+        if isinstance(query, list):
+            query = " ".join(query)
+
+        request_data: SerperSearchRequest = {
+            "q": query,
+        }
+        
+        if "max_results" in optional_params:
+            request_data["num"] = optional_params["max_results"]
+        
+        if "country" in optional_params:
+            request_data["gl"] = optional_params["country"].lower()
+        
+        if "search_domain_filter" in optional_params:
+            domains = optional_params["search_domain_filter"]
+            if isinstance(domains, list) and len(domains) > 0:
+                domain_clauses = " OR ".join(f"site:{d}" for d in domains)
+                request_data["q"] = f"({request_data['q']}) ({domain_clauses})"
+        
+        # Convert to dict before dynamic key assignments
+        result_data = dict(request_data)
+        
+        # pass through all other parameters as-is
+        for param, value in optional_params.items():
+            if param not in self.get_supported_perplexity_optional_params() and param not in result_data:
+                result_data[param] = value
+        
+        return result_data
+
+    def transform_search_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        **kwargs,
+    ) -> SearchResponse:
+        """
+        Transform Serper API response to LiteLLM unified SearchResponse format.
+        
+        Serper -> LiteLLM mappings:
+        - organic[].title -> SearchResult.title
+        - organic[].link  -> SearchResult.url
+        - organic[].snippet -> SearchResult.snippet
+        - organic[].date -> SearchResult.date (optional, not always present)
+        
+        Args:
+            raw_response: Raw httpx response from Serper API
+            logging_obj: Logging object for tracking
+            
+        Returns:
+            SearchResponse with standardized format
+        """
+        response_json = raw_response.json()
+        
+        results = []
+        for result in response_json.get("organic", []):
+            search_result = SearchResult(
+                title=result.get("title", ""),
+                url=result.get("link", ""),
+                snippet=result.get("snippet", ""),
+                date=result.get("date"),
+                last_updated=None,
+            )
+            results.append(search_result)
+        
+        return SearchResponse(
+            results=results,
+            object="search",
+        )
+

litellm/llms/vertex_ai/common_utils.py

@@ -571,38 +571,14 @@ def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]:
     return schema_dict
 
 
-def _is_any_type_schema(schema: dict) -> bool:
-    """
-    Detect schemas that represent "any JSON value" (no type constraints).
-
-    In JSON Schema, an empty schema {} means "any value is valid".
-    Schemas with only metadata keys (title, description, default, examples)
-    but no type-constraining keywords also represent "any type".
-
-    Gemini's Schema proto uses TYPE_UNSPECIFIED (0) as default,
-    so omitting the type field is valid and means "any type".
-    """
-    type_constraining_keys = {
-        "type",
-        "properties",
-        "items",
-        "anyOf",
-        "oneOf",
-        "allOf",
-        "enum",
-        "required",
-        "$ref",
-        "$schema",
-    }
-    return not any(key in type_constraining_keys for key in schema.keys())
-
-
 def process_items(schema, depth=0):
     if depth > DEFAULT_MAX_RECURSE_DEPTH:
         raise ValueError(
             f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
         )
     if isinstance(schema, dict):
+        if "items" in schema and schema["items"] == {}:
+            schema["items"] = {"type": "object"}
         for key, value in schema.items():
             if isinstance(value, dict):
                 process_items(value, depth + 1)
@@ -701,8 +677,9 @@ def convert_anyof_null_to_nullable(schema, depth=0):
                 # remove null type
                 anyof.remove(atype)
                 contains_null = True
-            elif isinstance(atype, dict) and _is_any_type_schema(atype):
-                pass  # preserve "any type" semantics — don't coerce to object
+            elif "type" not in atype and len(atype) == 0:
+                # Handle empty object case
+                atype["type"] = "object"
 
         if len(anyof) == 0:
             # Edge case: response schema with only null type present is invalid in Vertex AI
@@ -737,8 +714,7 @@ def add_object_type(schema):
     # Gemini requires all function parameters to be type OBJECT
     # Handle case where schema has no properties and no type (e.g. tools with no arguments)
     if "type" not in schema and "anyOf" not in schema and "oneOf" not in schema and "allOf" not in schema:
-        if not _is_any_type_schema(schema):
-            schema["type"] = "object"
+        schema["type"] = "object"
 
     properties = schema.get("properties", None)
     if properties is not None:

litellm/model_prices_and_context_window_backup.json

@@ -4207,6 +4207,41 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "azure/gpt-5.3-chat": {
+        "cache_read_input_token_cost": 1.75e-07,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "input_cost_per_token": 1.75e-06,
+        "input_cost_per_token_priority": 3.5e-06,
+        "litellm_provider": "azure",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_cost_per_token_priority": 2.8e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_service_tier": true,
+        "supports_vision": true
+    },
     "azure/gpt-5.3-codex": {
         "cache_read_input_token_cost": 1.75e-07,
         "input_cost_per_token": 1.75e-06,
@@ -4299,6 +4334,160 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "azure/gpt-5.4": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost_priority": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
+        "input_cost_per_token": 2.5e-06,
+        "input_cost_per_token_above_272k_tokens": 5e-06,
+        "input_cost_per_token_priority": 5e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 1e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "output_cost_per_token_above_272k_tokens": 2.25e-05,
+        "output_cost_per_token_priority": 3e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_service_tier": true,
+        "supports_vision": true
+    },
+    "azure/gpt-5.4-2026-03-05": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost_priority": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
+        "input_cost_per_token": 2.5e-06,
+        "input_cost_per_token_above_272k_tokens": 5e-06,
+        "input_cost_per_token_priority": 5e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 1e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "output_cost_per_token_above_272k_tokens": 2.25e-05,
+        "output_cost_per_token_priority": 3e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_service_tier": true,
+        "supports_vision": true
+    },
+    "azure/gpt-5.4-pro": {
+        "cache_read_input_token_cost": 3e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 6e-06,
+        "input_cost_per_token": 3e-05,
+        "input_cost_per_token_above_272k_tokens": 6e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 0.00018,
+        "output_cost_per_token_above_272k_tokens": 0.00027,
+        "supported_endpoints": [
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "azure/gpt-5.4-pro-2026-03-05": {
+        "cache_read_input_token_cost": 3e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 6e-06,
+        "input_cost_per_token": 3e-05,
+        "input_cost_per_token_above_272k_tokens": 6e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 0.00018,
+        "output_cost_per_token_above_272k_tokens": 0.00027,
+        "supported_endpoints": [
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "azure/gpt-image-1": {
         "cache_read_input_image_token_cost": 2.5e-06,
         "cache_read_input_token_cost": 1.25e-06,
@@ -12090,6 +12279,14 @@
             "notes": "SearXNG is an open-source metasearch engine. Free to use when self-hosted or using public instances."
         }
     },
+    "serper/search": {
+        "input_cost_per_query": 0.001,
+        "litellm_provider": "serper",
+        "mode": "search",
+        "metadata": {
+            "notes": "Serper Google Search API. Pricing: $1.00/1k queries (Starter), $0.75/1k (Standard), $0.50/1k (Scale), $0.30/1k (Ultimate)."
+        }
+    },
     "elevenlabs/scribe_v1": {
         "input_cost_per_second": 6.11e-05,
         "litellm_provider": "elevenlabs",
@@ -16799,6 +16996,42 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "gemini/gemini-3.1-flash-image-preview": {
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.045,
+        "output_cost_per_image_token": 6e-05,
+        "output_cost_per_image_token_batches": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "rpm": 1000,
+        "tpm": 4000000,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": false,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "gemini/deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,
@@ -21083,7 +21316,7 @@
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
-        "mode": "chat",
+        "mode": "responses",
         "output_cost_per_token": 0.00018,
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
@@ -21091,9 +21324,8 @@
         "output_cost_per_token_priority": 0.00027,
         "output_cost_per_token_above_272k_tokens_priority": 0.000405,
         "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/batch",
-            "/v1/responses"
+            "/v1/responses",
+            "/v1/batch"
         ],
         "supported_modalities": [
             "text",
@@ -21132,7 +21364,7 @@
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
-        "mode": "chat",
+        "mode": "responses",
         "output_cost_per_token": 0.00018,
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
@@ -21140,9 +21372,8 @@
         "output_cost_per_token_priority": 0.00027,
         "output_cost_per_token_above_272k_tokens_priority": 0.000405,
         "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/batch",
-            "/v1/responses"
+            "/v1/responses",
+            "/v1/batch"
         ],
         "supported_modalities": [
             "text",

litellm/provider_endpoints_support_backup.json

@@ -2061,6 +2061,13 @@
         "search": true
       }
     },
+    "serper": {
+      "display_name": "Serper (`serper`)",
+      "url": "https://docs.litellm.ai/docs/search/serper",
+      "endpoints": {
+        "search": true
+      }
+    },
     "triton": {
       "display_name": "Triton (`triton`)",
       "url": "https://docs.litellm.ai/docs/providers/triton-inference-server",

litellm/proxy/_types.py

@@ -202,6 +202,7 @@ class Litellm_EntityType(enum.Enum):
     ORGANIZATION = "organization"
     PROJECT = "project"
     TAG = "tag"
+    AGENT = "agent"
 
     # global proxy level entity
     PROXY = "proxy"
@@ -652,6 +653,8 @@ class LiteLLMRoutes(enum.Enum):
         "/model/update",
         "/model/delete",
         "/user/daily/activity",
+        "/user/available_roles",  # read-only role metadata; any authenticated user may read
+        "/user/list",  # org admins checked in endpoint; non-admins get 403
         "/model/{model_id}/update",
         "/prompt/list",
         "/prompt/info",
@@ -4228,6 +4231,7 @@ class DBSpendUpdateTransactions(TypedDict):
     team_member_list_transactions: Optional[Dict[str, float]]
     org_list_transactions: Optional[Dict[str, float]]
     tag_list_transactions: Optional[Dict[str, float]]
+    agent_list_transactions: Optional[Dict[str, float]]
 
 
 class SpendUpdateQueueItem(TypedDict, total=False):

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -39,7 +39,8 @@ def _jsonrpc_error(
 
 def _get_agent(agent_id: str):
     """Look up an agent by ID or name. Returns None if not found."""
-    from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+    from litellm.proxy.agent_endpoints.agent_registry import \
+        global_agent_registry
 
     agent = global_agent_registry.get_agent_by_id(agent_id=agent_id)
     if agent is None:
@@ -47,6 +48,26 @@ def _get_agent(agent_id: str):
     return agent
 
 
+def _enforce_inbound_trace_id(agent: Any, request: Request) -> None:
+    """Raise 400 if agent requires x-litellm-trace-id on inbound calls and it is missing."""
+    agent_litellm_params = agent.litellm_params or {}
+    if not agent_litellm_params.get("require_trace_id_on_calls_to_agent"):
+        return
+
+    from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers
+
+    headers_dict = dict(request.headers)
+    trace_id = get_chain_id_from_headers(headers_dict)
+    if not trace_id:
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                f"Agent '{agent.agent_id}' requires x-litellm-trace-id header "
+                "on all inbound requests."
+            ),
+        )
+
+
 async def _handle_stream_message(
     api_base: Optional[str],
     request_id: str,
@@ -116,9 +137,8 @@ async def _handle_stream_message(
                 and request_data is not None
                 and proxy_logging_obj is not None
             ):
-                from litellm.proxy.common_request_processing import (
-                    ProxyBaseLLMRequestProcessing,
-                )
+                from litellm.proxy.common_request_processing import \
+                    ProxyBaseLLMRequestProcessing
 
                 def _ndjson_chunk(chunk: Any) -> str:
                     if hasattr(chunk, "model_dump"):
@@ -218,9 +238,8 @@ async def get_agent_card(
     The URL in the agent card is rewritten to point to the LiteLLM proxy,
     so all subsequent A2A calls go through LiteLLM for logging and cost tracking.
     """
-    from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
-        AgentRequestHandler,
-    )
+    from litellm.proxy.agent_endpoints.auth.agent_permission_handler import \
+        AgentRequestHandler
 
     try:
         agent = _get_agent(agent_id)
@@ -284,15 +303,10 @@ async def invoke_agent_a2a(  # noqa: PLR0915
     """
     from litellm.a2a_protocol import asend_message
     from litellm.a2a_protocol.main import A2A_SDK_AVAILABLE
-    from litellm.proxy.agent_endpoints.auth.agent_permission_handler import (
-        AgentRequestHandler,
-    )
-    from litellm.proxy.proxy_server import (
-        general_settings,
-        proxy_config,
-        proxy_logging_obj,
-        version,
-    )
+    from litellm.proxy.agent_endpoints.auth.agent_permission_handler import \
+        AgentRequestHandler
+    from litellm.proxy.proxy_server import (general_settings, proxy_config,
+                                            proxy_logging_obj, version)
 
     body = {}
     try:
@@ -345,6 +359,8 @@ async def invoke_agent_a2a(  # noqa: PLR0915
                 detail=f"Agent '{agent_id}' is not allowed for your key/team. Contact proxy admin for access.",
             )
 
+        _enforce_inbound_trace_id(agent, request)
+
         # Get backend URL and agent name
         agent_url = agent.agent_card_params.get("url")
         agent_name = agent.agent_card_params.get("name", agent_id)
@@ -365,6 +381,10 @@ async def invoke_agent_a2a(  # noqa: PLR0915
         )
 
         # Set up data dict for litellm processing
+        if "metadata" not in body:
+            body["metadata"] = {}
+        body["metadata"]["agent_id"] = agent.agent_id
+
         body.update(
             {
                 "model": f"a2a_agent/{agent_name}",
@@ -373,9 +393,8 @@ async def invoke_agent_a2a(  # noqa: PLR0915
         )
 
         # Add litellm data (user_api_key, user_id, team_id, etc.)
-        from litellm.proxy.common_request_processing import (
-            ProxyBaseLLMRequestProcessing,
-        )
+        from litellm.proxy.common_request_processing import \
+            ProxyBaseLLMRequestProcessing
 
         processor = ProxyBaseLLMRequestProcessing(data=body)
         data, logging_obj = await processor.common_processing_pre_call_logic(

litellm/proxy/agent_endpoints/agent_registry.py

@@ -5,9 +5,8 @@ from typing import Any, Dict, List, Optional
 
 import litellm
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-from litellm.proxy.management_helpers.object_permission_utils import (
-    handle_update_object_permission_common,
-)
+from litellm.proxy.management_helpers.object_permission_utils import \
+    handle_update_object_permission_common
 from litellm.proxy.utils import PrismaClient
 from litellm.types.agents import AgentConfig, AgentResponse, PatchAgentRequest
 
@@ -152,6 +151,11 @@ class AgentRegistry:
             if object_permission_id is not None:
                 create_data["object_permission_id"] = object_permission_id
 
+            for rate_field in ("tpm_limit", "rpm_limit", "session_tpm_limit", "session_rpm_limit"):
+                _val = agent.get(rate_field)
+                if _val is not None:
+                    create_data[rate_field] = _val
+
             # Create agent in DB
             created_agent = await prisma_client.db.litellm_agentstable.create(
                 data=create_data,
@@ -226,6 +230,10 @@ class AgentRegistry:
                 update_data["agent_card_params"] = safe_dumps(
                     augment_agent.get("agent_card_params")
                 )
+
+            for rate_field in ("tpm_limit", "rpm_limit", "session_tpm_limit", "session_rpm_limit"):
+                if rate_field in agent:
+                    update_data[rate_field] = agent.get(rate_field)
             if "static_headers" in agent:
                 headers_value = agent.get("static_headers")
                 update_data["static_headers"] = safe_dumps(
@@ -321,6 +329,12 @@ class AgentRegistry:
                 "updated_by": updated_by,
                 "updated_at": datetime.now(timezone.utc),
             }
+
+            for rate_field in ("tpm_limit", "rpm_limit", "session_tpm_limit", "session_rpm_limit"):
+                _val = agent.get(rate_field)
+                if _val is not None:
+                    update_data[rate_field] = _val
+
             if agent.get("object_permission") is not None:
                 existing_agent = await prisma_client.db.litellm_agentstable.find_unique(
                     where={"agent_id": agent_id}

litellm/proxy/agent_endpoints/endpoints.py

@@ -8,12 +8,15 @@ Follows the A2A Spec.
 3. Get specific agent via GET `/v1/agents/{agent_id}`
 """
 
-from typing import Any, List, Optional
+import asyncio
+import os
+from typing import Any, Dict, List, Optional
 
-from fastapi import APIRouter, Depends, HTTPException, Request
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
@@ -25,6 +28,7 @@ from litellm.types.agents import (
     MakeAgentsPublicRequest,
     PatchAgentRequest,
 )
+from litellm.types.llms.custom_http import httpxSpecialProvider
 from litellm.types.proxy.management_endpoints.common_daily_activity import (
     SpendAnalyticsPaginatedResponse,
 )
@@ -49,6 +53,48 @@ def _check_agent_management_permission(user_api_key_dict: UserAPIKeyAuth) -> Non
         )
 
 
+AGENT_HEALTH_CHECK_TIMEOUT_SECONDS = float(
+    os.environ.get("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", "5.0")
+)
+AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS = float(
+    os.environ.get("LITELLM_AGENT_HEALTH_CHECK_GATHER_TIMEOUT", "30.0")
+)
+
+
+async def _check_agent_url_health(
+    agent: AgentResponse,
+) -> Dict[str, Any]:
+    """
+    Perform a GET request against the agent's URL and return the health result.
+
+    Returns a dict with ``agent_id``, ``healthy`` (bool), and an optional
+    ``error`` message.
+    """
+    url = (agent.agent_card_params or {}).get("url")
+    if not url:
+        return {"agent_id": agent.agent_id, "healthy": True}
+
+    try:
+        client = get_async_httpx_client(
+            llm_provider=httpxSpecialProvider.AgentHealthCheck,
+            params={"timeout": AGENT_HEALTH_CHECK_TIMEOUT_SECONDS},
+        )
+        response = await client.get(url)
+        if response.status_code >= 500:
+            return {
+                "agent_id": agent.agent_id,
+                "healthy": False,
+                "error": f"HTTP {response.status_code}",
+            }
+        return {"agent_id": agent.agent_id, "healthy": True}
+    except Exception as exc:
+        return {
+            "agent_id": agent.agent_id,
+            "healthy": False,
+            "error": str(exc),
+        }
+
+
 @router.get(
     "/v1/agents",
     tags=["[beta] A2A Agents"],
@@ -57,6 +103,10 @@ def _check_agent_management_permission(user_api_key_dict: UserAPIKeyAuth) -> Non
 )
 async def get_agents(
     request: Request,
+    health_check: bool = Query(
+        False,
+        description="When true, performs a GET request to each agent's URL. Agents with reachable URLs (HTTP status < 500) and agents without a URL are returned; unreachable agents are filtered out.",
+    ),
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),  # Used for auth
 ):
     """
@@ -67,6 +117,13 @@ async def get_agents(
       -H "Authorization: Bearer your-key" \
     ```
 
+    Pass `?health_check=true` to filter out agents whose URL is unreachable:
+    ```
+    curl -X GET "http://localhost:4000/v1/agents?health_check=true" \
+      -H "Content-Type: application/json" \
+      -H "Authorization: Bearer your-key" \
+    ```
+
     Returns: List[AgentResponse]
 
     """
@@ -79,7 +136,7 @@ async def get_agents(
 
     try:
         returned_agents: List[AgentResponse] = []
-        
+
         # Admin users get all agents
         if (
             user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
@@ -91,7 +148,7 @@ async def get_agents(
             allowed_agent_ids = await AgentRequestHandler.get_allowed_agents(
                 user_api_key_auth=user_api_key_dict
             )
-            
+
             # If no restrictions (empty list), return all agents
             if len(allowed_agent_ids) == 0:
                 returned_agents = global_agent_registry.get_agent_list()
@@ -99,10 +156,23 @@ async def get_agents(
                 # Filter agents by allowed IDs
                 all_agents = global_agent_registry.get_agent_list()
                 returned_agents = [
-                    agent for agent in all_agents
-                    if agent.agent_id in allowed_agent_ids
+                    agent for agent in all_agents if agent.agent_id in allowed_agent_ids
                 ]
 
+        # Fetch current spend from DB for all returned agents
+        from litellm.proxy.proxy_server import prisma_client
+
+        if prisma_client is not None:
+            agent_ids = [agent.agent_id for agent in returned_agents]
+            if agent_ids:
+                db_agents = await prisma_client.db.litellm_agentstable.find_many(
+                    where={"agent_id": {"in": agent_ids}},
+                )
+                spend_map = {a.agent_id: a.spend for a in db_agents}
+                for agent in returned_agents:
+                    if agent.agent_id in spend_map:
+                        agent.spend = spend_map[agent.agent_id]
+
         # add is_public field to each agent - we do it this way, to allow setting config agents as public
         for agent in returned_agents:
             if agent.litellm_params is None:
@@ -112,6 +182,44 @@ async def get_agents(
                 and (agent.agent_id in litellm.public_agent_groups)
             )
 
+        if health_check:
+            agents_with_url = [
+                agent
+                for agent in returned_agents
+                if (agent.agent_card_params or {}).get("url")
+            ]
+            agents_without_url = [
+                agent
+                for agent in returned_agents
+                if not (agent.agent_card_params or {}).get("url")
+            ]
+            try:
+                health_results = await asyncio.wait_for(
+                    asyncio.gather(
+                        *[_check_agent_url_health(agent) for agent in agents_with_url]
+                    ),
+                    timeout=AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS,
+                )
+            except asyncio.TimeoutError:
+                verbose_proxy_logger.warning(
+                    "Agent health check gather timed out after %s seconds",
+                    AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS,
+                )
+                health_results = [
+                    {"agent_id": agent.agent_id, "healthy": False, "error": "Health check timed out"}
+                    for agent in agents_with_url
+                ]
+            healthy_ids = {
+                result["agent_id"]
+                for result in health_results
+                if result["healthy"]
+            }
+            returned_agents = [
+                agent
+                for agent in agents_with_url
+                if agent.agent_id in healthy_ids
+            ] + agents_without_url
+
         return returned_agents
     except HTTPException:
         raise
@@ -128,9 +236,8 @@ async def get_agents(
 
 #### CRUD ENDPOINTS FOR AGENTS ####
 
-from litellm.proxy.agent_endpoints.agent_registry import (
-    global_agent_registry as AGENT_REGISTRY,
-)
+from litellm.proxy.agent_endpoints.agent_registry import \
+    global_agent_registry as AGENT_REGISTRY
 
 
 @router.post(
@@ -269,10 +376,21 @@ async def get_agent_by_id(
                 agent_dict = agent_row.model_dump()
                 if agent_row.object_permission is not None:
                     try:
-                        agent_dict["object_permission"] = agent_row.object_permission.model_dump()
+                        agent_dict["object_permission"] = (
+                            agent_row.object_permission.model_dump()
+                        )
                     except Exception:
-                        agent_dict["object_permission"] = agent_row.object_permission.dict()
+                        agent_dict["object_permission"] = (
+                            agent_row.object_permission.dict()
+                        )
                 agent = AgentResponse(**agent_dict)  # type: ignore
+        else:
+            # Agent found in memory — refresh spend from DB
+            db_row = await prisma_client.db.litellm_agentstable.find_unique(
+                where={"agent_id": agent_id}
+            )
+            if db_row is not None:
+                agent.spend = db_row.spend
 
         if agent is None:
             raise HTTPException(
@@ -580,9 +698,8 @@ async def make_agent_public(
     try:
         # Update the public model groups
         import litellm
-        from litellm.proxy.agent_endpoints.agent_registry import (
-            global_agent_registry as AGENT_REGISTRY,
-        )
+        from litellm.proxy.agent_endpoints.agent_registry import \
+            global_agent_registry as AGENT_REGISTRY
         from litellm.proxy.proxy_server import proxy_config
 
         # Check if user has admin permissions
@@ -697,9 +814,8 @@ async def make_agents_public(
     try:
         # Update the public model groups
         import litellm
-        from litellm.proxy.agent_endpoints.agent_registry import (
-            global_agent_registry as AGENT_REGISTRY,
-        )
+        from litellm.proxy.agent_endpoints.agent_registry import \
+            global_agent_registry as AGENT_REGISTRY
         from litellm.proxy.proxy_server import proxy_config
 
         # Load existing config
@@ -759,6 +875,7 @@ async def make_agents_public(
         verbose_proxy_logger.exception(f"Error making agent public: {e}")
         raise HTTPException(status_code=500, detail=str(e))
 
+
 @router.get(
     "/agent/daily/activity",
     tags=["Agent Management"],
@@ -820,4 +937,4 @@ async def get_agent_daily_activity(
         api_key=api_key,
         page=page,
         page_size=page_size,
-    )
+    )

litellm/proxy/auth/auth_checks.py

@@ -11,7 +11,8 @@ Run checks for:
 import asyncio
 import re
 import time
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast
+from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union,
+                    cast)
 
 from fastapi import HTTPException, Request, status
 from pydantic import BaseModel
@@ -20,48 +21,33 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.caching.dual_cache import LimitedSizeOrderedDict
-from litellm.constants import (
-    CLI_JWT_EXPIRATION_HOURS,
-    CLI_JWT_TOKEN_NAME,
-    DEFAULT_ACCESS_GROUP_CACHE_TTL,
-    DEFAULT_IN_MEMORY_TTL,
-    DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
-    DEFAULT_MAX_RECURSE_DEPTH,
-    EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE,
-)
+from litellm.constants import (CLI_JWT_EXPIRATION_HOURS, CLI_JWT_TOKEN_NAME,
+                               DEFAULT_ACCESS_GROUP_CACHE_TTL,
+                               DEFAULT_IN_MEMORY_TTL,
+                               DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
+                               DEFAULT_MAX_RECURSE_DEPTH,
+                               EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE)
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
-from litellm.proxy._types import (
-    RBAC_ROLES,
-    CallInfo,
-    LiteLLM_AccessGroupTable,
-    LiteLLM_BudgetTable,
-    LiteLLM_EndUserTable,
-    Litellm_EntityType,
-    LiteLLM_JWTAuth,
-    LiteLLM_ObjectPermissionTable,
-    LiteLLM_OrganizationMembershipTable,
-    LiteLLM_OrganizationTable,
-    LiteLLM_ProjectTableCachedObj,
-    LiteLLM_TagTable,
-    LiteLLM_TeamMembership,
-    LiteLLM_TeamTable,
-    LiteLLM_TeamTableCachedObj,
-    LiteLLM_UserTable,
-    LiteLLMRoutes,
-    LitellmUserRoles,
-    NewTeamRequest,
-    ProxyErrorTypes,
-    ProxyException,
-    RoleBasedPermissions,
-    SpecialModelNames,
-    UserAPIKeyAuth,
-)
+from litellm.proxy._types import (RBAC_ROLES, CallInfo,
+                                  LiteLLM_AccessGroupTable,
+                                  LiteLLM_BudgetTable, LiteLLM_EndUserTable,
+                                  Litellm_EntityType, LiteLLM_JWTAuth,
+                                  LiteLLM_ObjectPermissionTable,
+                                  LiteLLM_OrganizationMembershipTable,
+                                  LiteLLM_OrganizationTable,
+                                  LiteLLM_ProjectTableCachedObj,
+                                  LiteLLM_TagTable, LiteLLM_TeamMembership,
+                                  LiteLLM_TeamTable,
+                                  LiteLLM_TeamTableCachedObj,
+                                  LiteLLM_UserTable, LiteLLMRoutes,
+                                  LitellmUserRoles, NewTeamRequest,
+                                  ProxyErrorTypes, ProxyException,
+                                  RoleBasedPermissions, SpecialModelNames,
+                                  UserAPIKeyAuth)
 from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.proxy.guardrails.tool_name_extraction import (
-    TOOL_CAPABLE_CALL_TYPES,
-    extract_request_tool_names,
-)
+    TOOL_CAPABLE_CALL_TYPES, extract_request_tool_names)
 from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics
 from litellm.router import Router
@@ -224,6 +210,89 @@ async def _run_project_checks(
         )
 
 
+def _enforce_user_param_check(
+    general_settings: dict, request: Request, request_body: dict, route: str
+) -> None:
+    if not general_settings.get("enforce_user_param", False):
+        return
+
+    http_method = request.method if hasattr(request, "method") else None
+    is_post_method = http_method and http_method.upper() == "POST"
+    is_openai_route = RouteChecks.is_llm_api_route(route=route)
+    is_mcp_route = (
+        route in LiteLLMRoutes.mcp_routes.value
+        or RouteChecks.check_route_access(
+            route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value
+        )
+    )
+
+    if (
+        is_post_method
+        and is_openai_route
+        and not is_mcp_route
+        and "user" not in request_body
+    ):
+        raise Exception(
+            f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}"
+        )
+
+
+def _reject_clientside_metadata_tags_check(
+    general_settings: dict, request_body: dict, route: str
+) -> None:
+    if not general_settings.get("reject_clientside_metadata_tags", False):
+        return
+
+    if (
+        RouteChecks.is_llm_api_route(route=route)
+        and "metadata" in request_body
+        and isinstance(request_body["metadata"], dict)
+        and "tags" in request_body["metadata"]
+    ):
+        raise ProxyException(
+            message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via API key metadata.",
+            type=ProxyErrorTypes.bad_request_error,
+            param="metadata.tags",
+            code=status.HTTP_400_BAD_REQUEST,
+        )
+
+
+def _global_proxy_budget_check(
+    global_proxy_spend: Optional[float], skip_budget_checks: bool, route: str
+) -> None:
+    if (
+        litellm.max_budget > 0
+        and not skip_budget_checks
+        and global_proxy_spend is not None
+        and RouteChecks.is_llm_api_route(route=route)
+        and route != "/v1/models"
+        and route != "/models"
+    ):
+        if global_proxy_spend > litellm.max_budget:
+            raise litellm.BudgetExceededError(
+                current_cost=global_proxy_spend, max_budget=litellm.max_budget
+            )
+
+
+def _guardrail_modification_check(
+    request_body: dict, team_object: Optional[LiteLLM_TeamTable]
+) -> None:
+    _request_metadata: dict = request_body.get("metadata", {}) or {}
+    if not _request_metadata.get("guardrails"):
+        return
+
+    from litellm.proxy.guardrails.guardrail_helpers import \
+        can_modify_guardrails
+
+    if not can_modify_guardrails(team_object):
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "error": "Your team does not have permission to modify guardrails."
+            },
+        )
+
+
 async def check_tools_allowlist(
     request_body: dict,
     valid_token: Optional[UserAPIKeyAuth],
@@ -235,23 +304,34 @@ async def check_tools_allowlist(
     effective allowlist is read from valid_token.metadata and valid_token.team_metadata.
     Raises ProxyException with tool_access_denied if a tool is not allowed.
     """
-    from litellm.litellm_core_utils.api_route_to_call_types import (
-        get_call_types_for_route,
-    )
+    from litellm.litellm_core_utils.api_route_to_call_types import \
+        get_call_types_for_route
 
     if valid_token is None:
         return
     call_types = get_call_types_for_route(route)
-    if not call_types or not any(ct.value in TOOL_CAPABLE_CALL_TYPES for ct in call_types):
+    if not call_types or not any(
+        ct.value in TOOL_CAPABLE_CALL_TYPES for ct in call_types
+    ):
         return
     tool_names = extract_request_tool_names(route, request_body)
     if not tool_names:
         return
-    key_meta = (valid_token.metadata or {}) if isinstance(valid_token.metadata, dict) else {}
-    team_meta = (valid_token.team_metadata or {}) if isinstance(valid_token.team_metadata, dict) else {}
+    key_meta = (
+        (valid_token.metadata or {}) if isinstance(valid_token.metadata, dict) else {}
+    )
+    team_meta = (
+        (valid_token.team_metadata or {})
+        if isinstance(valid_token.team_metadata, dict)
+        else {}
+    )
     key_allowed = key_meta.get("allowed_tools")
     team_allowed = team_meta.get("allowed_tools")
-    effective = key_allowed if (isinstance(key_allowed, list) and len(key_allowed) > 0) else team_allowed
+    effective = (
+        key_allowed
+        if (isinstance(key_allowed, list) and len(key_allowed) > 0)
+        else team_allowed
+    )
     if not isinstance(effective, list) or len(effective) == 0:
         return
     allowed_set = {str(t) for t in effective}
@@ -326,6 +406,29 @@ async def common_checks(  # noqa: PLR0915
                 code=status.HTTP_401_UNAUTHORIZED,
             )
 
+    # Require trace id for agent keys when agent has require_trace_id_on_calls_by_agent
+    if valid_token is not None and valid_token.agent_id:
+        from litellm.proxy.agent_endpoints.agent_registry import \
+            global_agent_registry
+        from litellm.proxy.litellm_pre_call_utils import \
+            get_chain_id_from_headers
+
+        agent = global_agent_registry.get_agent_by_id(agent_id=valid_token.agent_id)
+        if agent is not None:
+            require_trace_id = (agent.litellm_params or {}).get(
+                "require_trace_id_on_calls_by_agent"
+            )
+            if require_trace_id:
+                headers_dict = dict(request.headers)
+                trace_id = get_chain_id_from_headers(headers_dict)
+                if not trace_id:
+                    raise ProxyException(
+                        message="Requests made with this agent's key must include the x-litellm-trace-id header.",
+                        type=ProxyErrorTypes.bad_request_error,
+                        param=None,
+                        code=status.HTTP_400_BAD_REQUEST,
+                    )
+
     ## 2.1 If user can call model (if personal key)
     if _model and team_object is None and user_object is not None:
         await can_user_call_model(
@@ -415,83 +518,10 @@ async def common_checks(  # noqa: PLR0915
                     message=f"ExceededBudget: End User={end_user_object.user_id} over budget. Spend={end_user_object.spend}, Budget={end_user_budget}",
                 )
 
-    # 6. [OPTIONAL] If 'enforce_user_param' enabled - did developer pass in 'user' param for openai endpoints
-    if (
-        general_settings.get("enforce_user_param", None) is not None
-        and general_settings["enforce_user_param"] is True
-    ):
-        # Get HTTP method from request
-        http_method = request.method if hasattr(request, "method") else None
-
-        # Check if it's a POST request and if it's an OpenAI route but not MCP
-        is_post_method = http_method and http_method.upper() == "POST"
-        is_openai_route = RouteChecks.is_llm_api_route(route=route)
-        is_mcp_route = (
-            route in LiteLLMRoutes.mcp_routes.value
-            or RouteChecks.check_route_access(
-                route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value
-            )
-        )
-
-        # Enforce user param only for POST requests on OpenAI routes (excluding MCP routes)
-        if (
-            is_post_method
-            and is_openai_route
-            and not is_mcp_route
-            and "user" not in request_body
-        ):
-            raise Exception(
-                f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}"
-            )
-
-    # 6.1 [OPTIONAL] If 'reject_clientside_metadata_tags' enabled - reject request if it has client-side 'metadata.tags'
-    if (
-        general_settings.get("reject_clientside_metadata_tags", None) is not None
-        and general_settings["reject_clientside_metadata_tags"] is True
-    ):
-        if (
-            RouteChecks.is_llm_api_route(route=route)
-            and "metadata" in request_body
-            and isinstance(request_body["metadata"], dict)
-            and "tags" in request_body["metadata"]
-        ):
-            raise ProxyException(
-                message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via API key metadata.",
-                type=ProxyErrorTypes.bad_request_error,
-                param="metadata.tags",
-                code=status.HTTP_400_BAD_REQUEST,
-            )
-    # 7. [OPTIONAL] If 'litellm.max_budget' is set (>0), is proxy under budget
-    if (
-        litellm.max_budget > 0
-        and not skip_budget_checks
-        and global_proxy_spend is not None
-        # only run global budget checks for OpenAI routes
-        # Reason - the Admin UI should continue working if the proxy crosses it's global budget
-        and RouteChecks.is_llm_api_route(route=route)
-        and route != "/v1/models"
-        and route != "/models"
-    ):
-        if global_proxy_spend > litellm.max_budget:
-            raise litellm.BudgetExceededError(
-                current_cost=global_proxy_spend, max_budget=litellm.max_budget
-            )
-
-    _request_metadata: dict = request_body.get("metadata", {}) or {}
-    if _request_metadata.get("guardrails"):
-        # check if team allowed to modify guardrails
-        from litellm.proxy.guardrails.guardrail_helpers import can_modify_guardrails
-
-        can_modify: bool = can_modify_guardrails(team_object)
-        if can_modify is False:
-            from fastapi import HTTPException
-
-            raise HTTPException(
-                status_code=403,
-                detail={
-                    "error": "Your team does not have permission to modify guardrails."
-                },
-            )
+    _enforce_user_param_check(general_settings, request, request_body, route)
+    _reject_clientside_metadata_tags_check(general_settings, request_body, route)
+    _global_proxy_budget_check(global_proxy_spend, skip_budget_checks, route)
+    _guardrail_modification_check(request_body, team_object)
 
     # 10 [OPTIONAL] Organization RBAC checks
     organization_role_based_access_check(
@@ -1932,9 +1962,8 @@ class ExperimentalUIJWTToken:
     def get_experimental_ui_login_jwt_auth_token(user_info: LiteLLM_UserTable) -> str:
         from datetime import timedelta
 
-        from litellm.proxy.common_utils.encrypt_decrypt_utils import (
-            encrypt_value_helper,
-        )
+        from litellm.proxy.common_utils.encrypt_decrypt_utils import \
+            encrypt_value_helper
 
         if user_info.user_role is None:
             raise Exception("User role is required for experimental UI login")
@@ -1980,9 +2009,8 @@ class ExperimentalUIJWTToken:
         """
         from datetime import timedelta
 
-        from litellm.proxy.common_utils.encrypt_decrypt_utils import (
-            encrypt_value_helper,
-        )
+        from litellm.proxy.common_utils.encrypt_decrypt_utils import \
+            encrypt_value_helper
 
         if user_info.user_role is None:
             raise Exception("User role is required for CLI JWT login")
@@ -2021,9 +2049,8 @@ class ExperimentalUIJWTToken:
         import json
 
         from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
-        from litellm.proxy.common_utils.encrypt_decrypt_utils import (
-            decrypt_value_helper,
-        )
+        from litellm.proxy.common_utils.encrypt_decrypt_utils import \
+            decrypt_value_helper
 
         decrypted_token = decrypt_value_helper(
             hashed_token, key="ui_hash_key", exception_type="debug"
@@ -2144,13 +2171,11 @@ async def get_key_object(
         )
 
     # else, check db
-    _valid_token: Optional[BaseModel] = (
-        await _fetch_key_object_from_db_with_reconnect(
-            hashed_token=hashed_token,
-            prisma_client=prisma_client,
-            parent_otel_span=parent_otel_span,
-            proxy_logging_obj=proxy_logging_obj,
-        )
+    _valid_token: Optional[BaseModel] = await _fetch_key_object_from_db_with_reconnect(
+        hashed_token=hashed_token,
+        prisma_client=prisma_client,
+        parent_otel_span=parent_otel_span,
+        proxy_logging_obj=proxy_logging_obj,
     )
 
     if _valid_token is None:
@@ -2296,9 +2321,9 @@ async def get_org_object(
         # Cache the result
         await user_api_key_cache.async_set_cache(
             key=cache_key,
-            value=response.model_dump()
-            if hasattr(response, "model_dump")
-            else response,
+            value=(
+                response.model_dump() if hasattr(response, "model_dump") else response
+            ),
             ttl=DEFAULT_IN_MEMORY_TTL,
         )
 
@@ -2341,8 +2366,10 @@ async def _get_resources_from_access_groups(
     # Lazy import to avoid circular imports
     if prisma_client is None or user_api_key_cache is None:
         from litellm.proxy.proxy_server import prisma_client as _prisma_client
-        from litellm.proxy.proxy_server import proxy_logging_obj as _proxy_logging_obj
-        from litellm.proxy.proxy_server import user_api_key_cache as _user_api_key_cache
+        from litellm.proxy.proxy_server import \
+            proxy_logging_obj as _proxy_logging_obj
+        from litellm.proxy.proxy_server import \
+            user_api_key_cache as _user_api_key_cache
 
         prisma_client = prisma_client or _prisma_client
         user_api_key_cache = user_api_key_cache or _user_api_key_cache
@@ -3298,7 +3325,8 @@ async def _tag_max_budget_check(
         BudgetExceededError if any tag is over its max budget.
         Triggers a budget alert if any tag is over its max budget.
     """
-    from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body
+    from litellm.proxy.common_utils.http_parsing_utils import \
+        get_tags_from_request_body
 
     if prisma_client is None:
         return

litellm/proxy/auth/auth_checks_organization.py

@@ -144,19 +144,32 @@ def _user_is_org_admin(
     user_object: Optional[LiteLLM_UserTable] = None,
 ) -> bool:
     """
-    Helper function to check if user is an org admin for the passed organization_id
-    """
-    if request_data.get("organization_id", None) is None:
-        return False
+    Helper function to check if user is an org admin for any of the passed organizations.
 
+    Checks both:
+    - `organization_id` (singular string) — legacy callers
+    - `organizations` (list of strings) — used by /user/new
+    """
     if user_object is None:
         return False
 
     if user_object.organization_memberships is None:
         return False
 
+    # Collect candidate org IDs from both fields
+    candidate_org_ids: List[str] = []
+    singular = request_data.get("organization_id", None)
+    if singular is not None:
+        candidate_org_ids.append(singular)
+    orgs_list = request_data.get("organizations", None)
+    if isinstance(orgs_list, list):
+        candidate_org_ids.extend(orgs_list)
+
+    if not candidate_org_ids:
+        return False
+
     for _membership in user_object.organization_memberships:
-        if _membership.organization_id == request_data.get("organization_id", None):
+        if _membership.organization_id in candidate_org_ids:
             if _membership.user_role == LitellmUserRoles.ORG_ADMIN.value:
                 return True
 

litellm/proxy/auth/user_api_key_auth.py

@@ -25,53 +25,38 @@ from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import (
-    ExperimentalUIJWTToken,
-    _cache_key_object,
-    _delete_cache_key_object,
-    _get_user_role,
-    _is_user_proxy_admin,
-    _virtual_key_max_budget_alert_check,
-    _virtual_key_max_budget_check,
-    _virtual_key_soft_budget_check,
-    can_key_call_model,
-    common_checks,
-    get_end_user_object,
-    get_jwt_key_mapping_object,
-    get_key_object,
-    get_project_object,
-    get_team_object,
-    get_user_object,
-    is_valid_fallback_model,
-)
-from litellm.proxy.auth.auth_exception_handler import UserAPIKeyAuthExceptionHandler
-from litellm.proxy.auth.auth_utils import (
-    abbreviate_api_key,
-    get_end_user_id_from_request_body,
-    get_model_from_request,
-    get_request_route,
-    normalize_request_route,
-    pre_db_read_auth_checks,
-    route_in_additonal_public_routes,
-)
+    ExperimentalUIJWTToken, _cache_key_object, _delete_cache_key_object,
+    _get_user_role, _is_user_proxy_admin, _virtual_key_max_budget_alert_check,
+    _virtual_key_max_budget_check, _virtual_key_soft_budget_check,
+    can_key_call_model, common_checks, get_end_user_object,
+    get_jwt_key_mapping_object, get_key_object, get_project_object,
+    get_team_object, get_user_object, is_valid_fallback_model)
+from litellm.proxy.auth.auth_exception_handler import \
+    UserAPIKeyAuthExceptionHandler
+from litellm.proxy.auth.auth_utils import (abbreviate_api_key,
+                                           get_end_user_id_from_request_body,
+                                           get_model_from_request,
+                                           get_request_route,
+                                           normalize_request_route,
+                                           pre_db_read_auth_checks,
+                                           route_in_additonal_public_routes)
 from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler
 from litellm.proxy.auth.oauth2_check import Oauth2Handler
 from litellm.proxy.auth.oauth2_proxy_hook import handle_oauth2_proxy_request
 from litellm.proxy.auth.route_checks import RouteChecks
-from litellm.proxy.common_utils.cache_coordinator import EventDrivenCacheCoordinator
+from litellm.proxy.common_utils.cache_coordinator import \
+    EventDrivenCacheCoordinator
 from litellm.proxy.common_utils.http_parsing_utils import (
-    _read_request_body,
-    _safe_get_request_headers,
-    populate_request_with_path_params,
-)
+    _read_request_body, _safe_get_request_headers,
+    populate_request_with_path_params)
 from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.services import ServiceTypes
 
 try:
-    from litellm_enterprise.proxy.auth.user_api_key_auth import (
-        enterprise_custom_auth as _enterprise_custom_auth,
-    )
+    from litellm_enterprise.proxy.auth.user_api_key_auth import \
+        enterprise_custom_auth as _enterprise_custom_auth
 
     enterprise_custom_auth: Optional[Callable] = _enterprise_custom_auth
 except ImportError as e:
@@ -351,9 +336,8 @@ def get_api_key(
         Tuple[Optional[str], Optional[str]]: Tuple of the api_key and the passed_in_key
     """
     from litellm.proxy.auth.route_checks import RouteChecks
-    from litellm.proxy.common_utils.http_parsing_utils import (
-        _safe_get_request_query_params,
-    )
+    from litellm.proxy.common_utils.http_parsing_utils import \
+        _safe_get_request_query_params
 
     api_key = api_key
     passed_in_key: Optional[str] = None
@@ -519,20 +503,15 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
     request_data: dict,
     custom_litellm_key_header: Optional[str] = None,
 ) -> UserAPIKeyAuth:
-    from litellm.proxy.proxy_server import (
-        general_settings,
-        jwt_handler,
-        litellm_proxy_admin_name,
-        llm_model_list,
-        llm_router,
-        master_key,
-        model_max_budget_limiter,
-        open_telemetry_logger,
-        prisma_client,
-        proxy_logging_obj,
-        user_api_key_cache,
-        user_custom_auth,
-    )
+    from litellm.proxy.proxy_server import (general_settings, jwt_handler,
+                                            litellm_proxy_admin_name,
+                                            llm_model_list, llm_router,
+                                            master_key,
+                                            model_max_budget_limiter,
+                                            open_telemetry_logger,
+                                            prisma_client, proxy_logging_obj,
+                                            user_api_key_cache,
+                                            user_custom_auth)
 
     parent_otel_span: Optional[Span] = None
     start_time = datetime.now()
@@ -636,17 +615,23 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
             # This allows UI SSO to work separately from API M2M authentication
             # Note: Info routes are already scoped to the user
             if RouteChecks.is_llm_api_route(route=route) or RouteChecks.is_info_route(route=route):
-                # return UserAPIKeyAuth object
-                # helper to check if the api_key is a valid oauth2 token
-                from litellm.proxy.proxy_server import premium_user
+                # When both OAuth2 and JWT auth are enabled, use token format to decide:
+                # - JWT tokens (3 dot-separated parts) -> skip OAuth2, fall through to JWT handler
+                # - Opaque tokens -> use OAuth2 handler
+                # This allows JWT for users and OAuth2 for M2M on the same instance
+                is_jwt_token = jwt_handler.is_jwt(token=api_key) if general_settings.get("enable_jwt_auth", False) is True else False
+                if not is_jwt_token:
+                    # return UserAPIKeyAuth object
+                    # helper to check if the api_key is a valid oauth2 token
+                    from litellm.proxy.proxy_server import premium_user
 
-                if premium_user is not True:
-                    raise ValueError(
-                        "Oauth2 token validation is only available for premium users"
-                        + CommonProxyErrors.not_premium_user.value
-                    )
+                    if premium_user is not True:
+                        raise ValueError(
+                            "Oauth2 token validation is only available for premium users"
+                            + CommonProxyErrors.not_premium_user.value
+                        )
 
-                return await Oauth2Handler.check_oauth2_token(token=api_key)
+                    return await Oauth2Handler.check_oauth2_token(token=api_key)
 
         if general_settings.get("enable_oauth2_proxy_auth", False) is True:
             return await handle_oauth2_proxy_request(request=request)
@@ -730,9 +715,11 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                                 if team_object is not None
                                 else None
                             ),
-                            team_metadata=team_object.metadata
-                            if team_object is not None
-                            else None,
+                            team_metadata=(
+                                team_object.metadata
+                                if team_object is not None
+                                else None
+                            ),
                             org_id=org_id,
                             end_user_id=end_user_id,
                             parent_otel_span=parent_otel_span,
@@ -750,9 +737,9 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                         team_rpm_limit=(
                             team_object.rpm_limit if team_object is not None else None
                         ),
-                        team_models=team_object.models
-                        if team_object is not None
-                        else [],
+                        team_models=(
+                            team_object.models if team_object is not None else []
+                        ),
                         user_role=(
                             LitellmUserRoles(user_object.user_role)
                             if user_object is not None
@@ -779,16 +766,17 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                             if team_membership is not None
                             else None
                         ),
-                        team_metadata=team_object.metadata
-                        if team_object is not None
-                        else None,
+                        team_metadata=(
+                            team_object.metadata if team_object is not None else None
+                        ),
                     )
 
                     # Check if model has zero cost - if so, skip all budget checks
                     model = get_model_from_request(request_data, route)
                     skip_budget_checks = False
                     if model is not None and llm_router is not None:
-                        from litellm.proxy.auth.auth_checks import _is_model_cost_zero
+                        from litellm.proxy.auth.auth_checks import \
+                            _is_model_cost_zero
 
                         skip_budget_checks = _is_model_cost_zero(
                             model=model, llm_router=llm_router
@@ -893,9 +881,9 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                     route=route,
                 )
                 if _end_user_object is not None:
-                    end_user_params[
-                        "allowed_model_region"
-                    ] = _end_user_object.allowed_model_region
+                    end_user_params["allowed_model_region"] = (
+                        _end_user_object.allowed_model_region
+                    )
                     if _end_user_object.litellm_budget_table is not None:
                         _apply_budget_limits_to_end_user_params(
                             end_user_params=end_user_params,
@@ -904,9 +892,8 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                         )
                 elif litellm.max_end_user_budget_id is not None:
                     # End user doesn't exist yet, but apply default budget limits if configured
-                    from litellm.proxy.auth.auth_checks import (
-                        get_default_end_user_budget,
-                    )
+                    from litellm.proxy.auth.auth_checks import \
+                        get_default_end_user_budget
 
                     default_budget = await get_default_end_user_budget(
                         prisma_client=prisma_client,
@@ -1463,9 +1450,9 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
 
             if _end_user_object is not None:
                 valid_token_dict.update(end_user_params)
-                valid_token_dict[
-                    "end_user_object_permission"
-                ] = _end_user_object.object_permission
+                valid_token_dict["end_user_object_permission"] = (
+                    _end_user_object.object_permission
+                )
 
         # check if token is from litellm-ui, litellm ui makes keys to allow users to login with sso. These keys can only be used for LiteLLM UI functions
         # sso/login, ui/login, /key functions and /user functions
@@ -1687,7 +1674,8 @@ async def _lookup_end_user_and_apply_budget(
                 valid_token=valid_token, end_user_params=end_user_params
             )
         elif litellm.max_end_user_budget_id is not None:
-            from litellm.proxy.auth.auth_checks import get_default_end_user_budget
+            from litellm.proxy.auth.auth_checks import \
+                get_default_end_user_budget
 
             default_budget = await get_default_end_user_budget(
                 prisma_client=prisma_client,
@@ -1718,14 +1706,10 @@ async def _run_post_custom_auth_checks(
     route: str,
     parent_otel_span: Optional[Span],
 ) -> UserAPIKeyAuth:
-    from litellm.proxy.proxy_server import (
-        prisma_client,
-        user_api_key_cache,
-        proxy_logging_obj,
-        general_settings,
-        llm_router,
-        model_max_budget_limiter,
-    )
+    from litellm.proxy.proxy_server import (general_settings, llm_router,
+                                            model_max_budget_limiter,
+                                            prisma_client, proxy_logging_obj,
+                                            user_api_key_cache)
 
     # 1. Look up end_user object from DB if end_user_id is set
     end_user_object = None
@@ -1756,9 +1740,11 @@ async def _run_post_custom_auth_checks(
                 message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}",
                 type=ProxyErrorTypes.expired_key,
                 code=400,
-                param=abbreviate_api_key(api_key=valid_token.token)
-                if valid_token.token
-                else "",
+                param=(
+                    abbreviate_api_key(api_key=valid_token.token)
+                    if valid_token.token
+                    else ""
+                ),
             )
 
     current_model = request_data.get("model", None)

litellm/proxy/batches_endpoints/endpoints.py

@@ -27,6 +27,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     encode_file_id_with_model,
     get_batch_from_database,
     get_credentials_for_model,
+    get_model_id_from_unified_batch_id,
     get_models_from_unified_file_id,
     get_original_file_id,
     prepare_data_with_credentials,
@@ -487,6 +488,10 @@ async def retrieve_batch( # noqa: PLR0915
 
             response = await llm_router.aretrieve_batch(**data)  # type: ignore
             response._hidden_params["unified_batch_id"] = unified_batch_id
+            if unified_batch_id:
+                model_id_from_batch = get_model_id_from_unified_batch_id(unified_batch_id)
+                if model_id_from_batch:
+                    response._hidden_params["model_id"] = model_id_from_batch
         
         # SCENARIO 3: Fallback to custom_llm_provider (uses env variables)
         else:

litellm/proxy/db/db_spend_update_writer.py

@@ -13,36 +13,49 @@ import random
 import time
 import traceback
 from datetime import datetime, timedelta, timezone
-from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union,
-                    cast, overload)
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Union,
+    cast,
+    overload,
+)
 
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching import DualCache, RedisCache
 from litellm.constants import DB_SPEND_UPDATE_JOB_NAME
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.proxy._types import (DB_CONNECTION_ERROR_TYPES,
-                                  BaseDailySpendTransaction,
-                                  DailyAgentSpendTransaction,
-                                  DailyEndUserSpendTransaction,
-                                  DailyOrganizationSpendTransaction,
-                                  DailyTagSpendTransaction,
-                                  DailyTeamSpendTransaction,
-                                  DailyUserSpendTransaction,
-                                  DBSpendUpdateTransactions,
-                                  Litellm_EntityType, LiteLLM_UserTable,
-                                  SpendLogsMetadata, SpendLogsPayload,
-                                  SpendUpdateQueueItem, ToolDiscoveryQueueItem)
-from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import \
-    DailySpendUpdateQueue
-from litellm.proxy.db.db_transaction_queue.pod_lock_manager import \
-    PodLockManager
-from litellm.proxy.db.db_transaction_queue.redis_update_buffer import \
-    RedisUpdateBuffer
-from litellm.proxy.db.db_transaction_queue.spend_update_queue import \
-    SpendUpdateQueue
-from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import \
-    ToolDiscoveryQueue
+from litellm.proxy._types import (
+    DB_CONNECTION_ERROR_TYPES,
+    BaseDailySpendTransaction,
+    DailyAgentSpendTransaction,
+    DailyEndUserSpendTransaction,
+    DailyOrganizationSpendTransaction,
+    DailyTagSpendTransaction,
+    DailyTeamSpendTransaction,
+    DailyUserSpendTransaction,
+    DBSpendUpdateTransactions,
+    Litellm_EntityType,
+    LiteLLM_UserTable,
+    SpendLogsMetadata,
+    SpendLogsPayload,
+    SpendUpdateQueueItem,
+    ToolDiscoveryQueueItem,
+)
+from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
+    DailySpendUpdateQueue,
+)
+from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
+from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer
+from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue
+from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
+    ToolDiscoveryQueue,
+)
 from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
 
 if TYPE_CHECKING:
@@ -91,10 +104,12 @@ class DBSpendUpdateWriter:
         end_time: Optional[datetime],
         response_cost: Optional[float],
     ):
-        from litellm.proxy.proxy_server import (disable_spend_logs,
-                                                litellm_proxy_budget_name,
-                                                prisma_client,
-                                                user_api_key_cache)
+        from litellm.proxy.proxy_server import (
+            disable_spend_logs,
+            litellm_proxy_budget_name,
+            prisma_client,
+            user_api_key_cache,
+        )
         from litellm.proxy.utils import ProxyUpdateSpend, hash_token
 
         try:
@@ -109,8 +124,9 @@ class DBSpendUpdateWriter:
                 hashed_token = token
 
             ## CREATE SPEND LOG PAYLOAD ##
-            from litellm.proxy.spend_tracking.spend_tracking_utils import \
-                get_logging_payload
+            from litellm.proxy.spend_tracking.spend_tracking_utils import (
+                get_logging_payload,
+            )
 
             payload = get_logging_payload(
                 kwargs=kwargs,
@@ -374,6 +390,19 @@ class DBSpendUpdateWriter:
                 traceback.format_exc(),
             )
 
+        _agent_id_for_spend = payload_copy.get("agent_id")
+        try:
+            await self._update_agent_db(
+                response_cost=response_cost,
+                agent_id=_agent_id_for_spend,
+                prisma_client=prisma_client,
+            )
+        except Exception:
+            verbose_proxy_logger.debug(
+                "_batch_database_updates: _update_agent_db failed: %s",
+                traceback.format_exc(),
+            )
+
         try:
             await self.add_spend_log_transaction_to_daily_user_transaction(
                 payload=payload_copy,
@@ -604,6 +633,34 @@ class DBSpendUpdateWriter:
             )
             raise e
 
+    async def _update_agent_db(
+        self,
+        response_cost: Optional[float],
+        agent_id: Optional[str],
+        prisma_client: Optional[PrismaClient],
+    ):
+        try:
+            if agent_id is None or prisma_client is None:
+                return
+
+            await self.spend_update_queue.add_update(
+                update=SpendUpdateQueueItem(
+                    entity_type=Litellm_EntityType.AGENT,
+                    entity_id=agent_id,
+                    response_cost=response_cost,
+                )
+            )
+        except Exception as e:
+            verbose_proxy_logger.error(
+                "Spend tracking - failed to enqueue agent spend update. "
+                "agent_id=%s, response_cost=%s - %s\n%s",
+                agent_id,
+                response_cost,
+                str(e),
+                traceback.format_exc(),
+            )
+            raise e
+
     async def _update_tag_db(
         self,
         response_cost: Optional[float],
@@ -765,7 +822,7 @@ class DBSpendUpdateWriter:
                 if db_spend_update_transactions is not None:
                     verbose_proxy_logger.info(
                         "Spend tracking - committing spend updates from Redis to DB: "
-                        "keys=%d, users=%d, teams=%d, orgs=%d, end_users=%d, team_members=%d, tags=%d",
+                        "keys=%d, users=%d, teams=%d, orgs=%d, end_users=%d, team_members=%d, tags=%d, agents=%d",
                         len(
                             db_spend_update_transactions.get("key_list_transactions")
                             or {}
@@ -798,6 +855,12 @@ class DBSpendUpdateWriter:
                             db_spend_update_transactions.get("tag_list_transactions")
                             or {}
                         ),
+                        len(
+                            db_spend_update_transactions.get(
+                                "agent_list_transactions"
+                            )
+                            or {}
+                        ),
                     )
                     await self._commit_spend_updates_to_db(
                         prisma_client=prisma_client,
@@ -1002,8 +1065,10 @@ class DBSpendUpdateWriter:
         Commits all the spend `UPDATE` transactions to the Database
 
         """
-        from litellm.proxy.utils import (ProxyUpdateSpend,
-                                         _raise_failed_update_spend_exception)
+        from litellm.proxy.utils import (
+            ProxyUpdateSpend,
+            _raise_failed_update_spend_exception,
+        )
 
         ### UPDATE USER TABLE ###
         user_list_transactions = db_spend_update_transactions["user_list_transactions"]
@@ -1279,6 +1344,18 @@ class DBSpendUpdateWriter:
             proxy_logging_obj=proxy_logging_obj,
         )
 
+        ### UPDATE AGENT TABLE ###
+        agent_list_transactions = db_spend_update_transactions["agent_list_transactions"]
+        await DBSpendUpdateWriter._update_entity_spend_in_db(
+            entity_name="Agent",
+            transactions=agent_list_transactions,
+            table_accessor="litellm_agentstable",
+            where_field="agent_id",
+            n_retry_times=n_retry_times,
+            prisma_client=prisma_client,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+
     @staticmethod
     async def _update_entity_spend_in_db(
         entity_name: str,
@@ -2031,9 +2108,6 @@ class DBSpendUpdateWriter:
             )
             return
         if payload["agent_id"] is None:
-            verbose_proxy_logger.debug(
-                "agent_id is None for request. Skipping incrementing agent spend."
-            )
             return
         payload_with_agent_id = cast(
             SpendLogsPayload,

litellm/proxy/db/db_transaction_queue/redis_update_buffer.py

@@ -10,33 +10,31 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm.caching import RedisCache
-from litellm.constants import (
-    MAX_REDIS_BUFFER_DEQUEUE_COUNT,
-    REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY,
-    REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY,
-    REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY,
-    REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY,
-    REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY,
-    REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY,
-    REDIS_UPDATE_BUFFER_KEY,
-)
+from litellm.constants import (MAX_REDIS_BUFFER_DEQUEUE_COUNT,
+                               REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY,
+                               REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY,
+                               REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY,
+                               REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY,
+                               REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY,
+                               REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY,
+                               REDIS_UPDATE_BUFFER_KEY)
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-from litellm.proxy._types import (
-    DailyTagSpendTransaction,
-    DailyTeamSpendTransaction,
-    DailyUserSpendTransaction,
-    DailyOrganizationSpendTransaction,
-    DailyEndUserSpendTransaction,
-    DBSpendUpdateTransactions,
-    DailyAgentSpendTransaction,
-)
-from litellm.proxy.db.db_transaction_queue.base_update_queue import service_logger_obj
-from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
-    DailySpendUpdateQueue,
-)
-from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue
+from litellm.proxy._types import (DailyAgentSpendTransaction,
+                                  DailyEndUserSpendTransaction,
+                                  DailyOrganizationSpendTransaction,
+                                  DailyTagSpendTransaction,
+                                  DailyTeamSpendTransaction,
+                                  DailyUserSpendTransaction,
+                                  DBSpendUpdateTransactions)
+from litellm.proxy.db.db_transaction_queue.base_update_queue import \
+    service_logger_obj
+from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import \
+    DailySpendUpdateQueue
+from litellm.proxy.db.db_transaction_queue.spend_update_queue import \
+    SpendUpdateQueue
 from litellm.secret_managers.main import str_to_bool
-from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation
+from litellm.types.caching import (RedisPipelineLpopOperation,
+                                   RedisPipelineRpushOperation)
 from litellm.types.services import ServiceTypes
 
 if TYPE_CHECKING:
@@ -579,6 +577,7 @@ class RedisUpdateBuffer:
             team_member_list_transactions={},
             org_list_transactions={},
             tag_list_transactions={},
+            agent_list_transactions={},
         )
 
         # Define the transaction fields to process
@@ -590,6 +589,7 @@ class RedisUpdateBuffer:
             "team_member_list_transactions",
             "org_list_transactions",
             "tag_list_transactions",
+            "agent_list_transactions",
         ]
 
         # Loop through each transaction and combine the values

litellm/proxy/db/db_transaction_queue/spend_update_queue.py

@@ -3,15 +3,10 @@ from typing import Dict, List, Optional
 
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import LITELLM_ASYNCIO_QUEUE_MAXSIZE
-from litellm.proxy._types import (
-    DBSpendUpdateTransactions,
-    Litellm_EntityType,
-    SpendUpdateQueueItem,
-)
+from litellm.proxy._types import (DBSpendUpdateTransactions,
+                                  Litellm_EntityType, SpendUpdateQueueItem)
 from litellm.proxy.db.db_transaction_queue.base_update_queue import (
-    BaseUpdateQueue,
-    service_logger_obj,
-)
+    BaseUpdateQueue, service_logger_obj)
 from litellm.types.services import ServiceTypes
 
 
@@ -145,6 +140,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
             team_member_list_transactions={},
             org_list_transactions={},
             tag_list_transactions={},
+            agent_list_transactions={},
         )
 
         # Map entity types to their corresponding transaction dictionary keys
@@ -156,6 +152,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
             Litellm_EntityType.TEAM_MEMBER: "team_member_list_transactions",
             Litellm_EntityType.ORGANIZATION: "org_list_transactions",
             Litellm_EntityType.TAG: "tag_list_transactions",
+            Litellm_EntityType.AGENT: "agent_list_transactions",
         }
 
         for update in updates:
@@ -207,6 +204,10 @@ class SpendUpdateQueue(BaseUpdateQueue):
                 transactions_dict = db_spend_update_transactions[
                     "tag_list_transactions"
                 ]
+            elif dict_key == "agent_list_transactions":
+                transactions_dict = db_spend_update_transactions[
+                    "agent_list_transactions"
+                ]
             else:
                 continue
 

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py

@@ -341,6 +341,30 @@ class GenericGuardrailAPI(CustomGuardrail):
             return_inputs["tools"] = tools
         return return_inputs
 
+    def _handle_guardrail_request_error(
+        self,
+        error: Exception,
+        inputs: GenericGuardrailAPIInputs,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional["LiteLLMLoggingObj"],
+        is_unreachable: bool = True,
+    ) -> GenericGuardrailAPIInputs:
+        if is_unreachable and self.unreachable_fallback == "fail_open":
+            http_status_code = getattr(
+                getattr(error, "response", None), "status_code", None
+            )
+            return self._fail_open_passthrough(
+                inputs=inputs,
+                input_type=input_type,
+                logging_obj=logging_obj,
+                error=error,
+                **({"http_status_code": http_status_code} if http_status_code else {}),
+            )
+        verbose_proxy_logger.error(
+            "Generic Guardrail API: failed to make request: %s", str(error)
+        )
+        raise Exception(f"Generic Guardrail API failed: {str(error)}")
+
     @log_guardrail_information
     async def apply_guardrail(
         self,
@@ -466,58 +490,24 @@ class GenericGuardrailAPI(CustomGuardrail):
             )
 
         except GuardrailRaisedException:
-            # Re-raise guardrail exceptions as-is
             raise
         except Timeout as e:
-            # AsyncHTTPHandler wraps httpx.TimeoutException into litellm.Timeout
-            if self.unreachable_fallback == "fail_open":
-                return self._fail_open_passthrough(
-                    inputs=inputs,
-                    input_type=input_type,
-                    logging_obj=logging_obj,
-                    error=e,
-                )
-
-            verbose_proxy_logger.error(
-                "Generic Guardrail API: failed to make request: %s", str(e)
+            return self._handle_guardrail_request_error(
+                e, inputs, input_type, logging_obj
             )
-            raise Exception(f"Generic Guardrail API failed: {str(e)}")
         except httpx.HTTPStatusError as e:
-            # Common reverse-proxy/LB failures can present as HTTP errors even when the backend is unreachable.
-            status_code = getattr(getattr(e, "response", None), "status_code", None)
-            if self.unreachable_fallback == "fail_open" and status_code in (
-                502,
-                503,
-                504,
-            ):
-                return self._fail_open_passthrough(
-                    inputs=inputs,
-                    input_type=input_type,
-                    logging_obj=logging_obj,
-                    error=e,
-                    http_status_code=status_code,
-                )
-
-            verbose_proxy_logger.error(
-                "Generic Guardrail API: failed to make request: %s", str(e)
+            status_code = getattr(
+                getattr(e, "response", None), "status_code", None
+            )
+            is_unreachable = status_code in (502, 503, 504)
+            return self._handle_guardrail_request_error(
+                e, inputs, input_type, logging_obj, is_unreachable=is_unreachable
             )
-            raise Exception(f"Generic Guardrail API failed: {str(e)}")
         except httpx.RequestError as e:
-            # Guardrail endpoint is unreachable (DNS/connect/timeout/etc)
-            if self.unreachable_fallback == "fail_open":
-                return self._fail_open_passthrough(
-                    inputs=inputs,
-                    input_type=input_type,
-                    logging_obj=logging_obj,
-                    error=e,
-                )
-
-            verbose_proxy_logger.error(
-                "Generic Guardrail API: failed to make request: %s", str(e)
+            return self._handle_guardrail_request_error(
+                e, inputs, input_type, logging_obj
             )
-            raise Exception(f"Generic Guardrail API failed: {str(e)}")
         except Exception as e:
-            verbose_proxy_logger.error(
-                "Generic Guardrail API: failed to make request: %s", str(e)
+            return self._handle_guardrail_request_error(
+                e, inputs, input_type, logging_obj, is_unreachable=False
             )
-            raise Exception(f"Generic Guardrail API failed: {str(e)}")

litellm/proxy/hooks/__init__.py

@@ -5,6 +5,8 @@ from . import *
 from .cache_control_check import _PROXY_CacheControlCheck
 from .litellm_skills import SkillsInjectionHook
 from .max_budget_limiter import _PROXY_MaxBudgetLimiter
+from .max_budget_per_session_limiter import _PROXY_MaxBudgetPerSessionHandler
+from .max_iterations_limiter import _PROXY_MaxIterationsHandler
 from .parallel_request_limiter import _PROXY_MaxParallelRequestsHandler
 from .parallel_request_limiter_v3 import _PROXY_MaxParallelRequestsHandler_v3
 from .responses_id_security import ResponsesIDSecurity
@@ -23,6 +25,8 @@ PROXY_HOOKS = {
     "cache_control_check": _PROXY_CacheControlCheck,
     "responses_id_security": ResponsesIDSecurity,
     "litellm_skills": SkillsInjectionHook,
+    "max_iterations_limiter": _PROXY_MaxIterationsHandler,
+    "max_budget_per_session_limiter": _PROXY_MaxBudgetPerSessionHandler,
 }
 
 ## FEATURE FLAG HOOKS ##

litellm/proxy/hooks/max_budget_per_session_limiter.py

@@ -0,0 +1,271 @@
+"""
+Per-Session Budget Limiter for LiteLLM Proxy.
+
+Enforces a dollar-amount cap per session (identified by `session_id` /
+`x-litellm-trace-id`). After each successful LLM call the response cost is
+accumulated against the session. When the accumulated spend exceeds
+`max_budget_per_session` (configured in agent litellm_params), subsequent
+requests for that session receive a 429.
+
+Note: trace-id enforcement (require_trace_id_on_calls_by_agent) is handled
+separately in auth_checks.py at the agent level, not in this hook.
+
+Works across multiple proxy instances via DualCache (in-memory + Redis).
+Follows the same pattern as max_iterations_limiter.py.
+"""
+
+import os
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from fastapi import HTTPException
+
+from litellm import DualCache
+from litellm._logging import verbose_proxy_logger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.proxy._types import UserAPIKeyAuth
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
+
+    InternalUsageCache = _InternalUsageCache
+else:
+    InternalUsageCache = Any
+
+
+# Redis Lua script for atomic float increment with TTL.
+# INCRBYFLOAT returns the new value as a string.
+# Only sets EXPIRE on first call (when prior value was nil).
+MAX_BUDGET_SESSION_INCREMENT_SCRIPT = """
+local key = KEYS[1]
+local amount = ARGV[1]
+local ttl = tonumber(ARGV[2])
+
+local existed = redis.call('EXISTS', key)
+local new_val = redis.call('INCRBYFLOAT', key, amount)
+if existed == 0 then
+    redis.call('EXPIRE', key, ttl)
+end
+
+return new_val
+"""
+
+# Default TTL for session budget counters (1 hour)
+DEFAULT_MAX_BUDGET_PER_SESSION_TTL = 3600
+
+
+class _PROXY_MaxBudgetPerSessionHandler(CustomLogger):
+    """
+    Pre-call hook that enforces max_budget_per_session.
+
+    Configuration (set in agent litellm_params):
+        - max_budget_per_session: dollar cap per session_id
+
+    Cache key pattern:
+        {session_budget:<session_id>}:spend
+    """
+
+    def __init__(self, internal_usage_cache: InternalUsageCache):
+        self.internal_usage_cache = internal_usage_cache
+        self.ttl = int(
+            os.getenv(
+                "LITELLM_MAX_BUDGET_PER_SESSION_TTL",
+                DEFAULT_MAX_BUDGET_PER_SESSION_TTL,
+            )
+        )
+
+        if self.internal_usage_cache.dual_cache.redis_cache is not None:
+            self.increment_script = (
+                self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
+                    MAX_BUDGET_SESSION_INCREMENT_SCRIPT
+                )
+            )
+        else:
+            self.increment_script = None
+
+    async def async_pre_call_hook(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        cache: DualCache,
+        data: dict,
+        call_type: str,
+    ) -> Optional[Union[Exception, str, dict]]:
+        """
+        Before each LLM call, check if max_budget_per_session is set and
+        whether accumulated spend exceeds the budget (429 if so).
+        """
+        max_budget = self._get_max_budget_per_session(user_api_key_dict)
+
+        session_id = self._get_session_id(data)
+
+        if max_budget is None or session_id is None:
+            return None
+
+        max_budget = float(max_budget)
+        cache_key = self._make_cache_key(session_id)
+        current_spend = await self._get_current_spend(cache_key)
+
+        verbose_proxy_logger.debug(
+            "MaxBudgetPerSessionHandler: session_id=%s, spend=%.4f, max=%.2f",
+            session_id,
+            current_spend,
+            max_budget,
+        )
+
+        if current_spend >= max_budget:
+            raise HTTPException(
+                status_code=429,
+                detail=(
+                    f"Session budget exceeded for session {session_id}. "
+                    f"Current spend: ${current_spend:.4f}, "
+                    f"max_budget_per_session: ${max_budget:.2f}."
+                ),
+            )
+
+        return None
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        """
+        After a successful LLM call, increment the session spend by the response cost.
+        """
+        try:
+            litellm_params = kwargs.get("litellm_params") or {}
+            metadata = litellm_params.get("metadata") or {}
+            session_id = metadata.get("session_id")
+            if session_id is None:
+                return
+
+            agent_id = metadata.get("agent_id")
+            if agent_id is None:
+                return
+
+            from litellm.proxy.agent_endpoints.agent_registry import (
+                global_agent_registry,
+            )
+
+            agent = global_agent_registry.get_agent_by_id(agent_id=str(agent_id))
+            if agent is None:
+                return
+
+            agent_litellm_params = agent.litellm_params or {}
+            max_budget = agent_litellm_params.get("max_budget_per_session")
+            if max_budget is None:
+                return
+
+            response_cost = kwargs.get("response_cost") or 0.0
+            if response_cost <= 0:
+                return
+
+            cache_key = self._make_cache_key(str(session_id))
+            await self._increment_spend(cache_key, float(response_cost))
+
+            verbose_proxy_logger.debug(
+                "MaxBudgetPerSessionHandler: incremented session %s spend by %.6f",
+                session_id,
+                response_cost,
+            )
+        except Exception as e:
+            verbose_proxy_logger.warning(
+                "MaxBudgetPerSessionHandler: error in async_log_success_event: %s",
+                str(e),
+            )
+
+    def _get_session_id(self, data: dict) -> Optional[str]:
+        """Extract session_id from request metadata."""
+        metadata = data.get("metadata") or {}
+        session_id = metadata.get("session_id")
+        if session_id is not None:
+            return str(session_id)
+
+        litellm_metadata = data.get("litellm_metadata") or {}
+        session_id = litellm_metadata.get("session_id")
+        if session_id is not None:
+            return str(session_id)
+
+        return None
+
+    def _get_max_budget_per_session(
+        self, user_api_key_dict: UserAPIKeyAuth
+    ) -> Optional[float]:
+        """Extract max_budget_per_session from agent litellm_params."""
+        agent_id = user_api_key_dict.agent_id
+        if agent_id is None:
+            return None
+
+        from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+
+        agent = global_agent_registry.get_agent_by_id(agent_id=agent_id)
+        if agent is None:
+            return None
+
+        litellm_params = agent.litellm_params or {}
+        max_budget = litellm_params.get("max_budget_per_session")
+        if max_budget is not None:
+            return float(max_budget)
+        return None
+
+    def _make_cache_key(self, session_id: str) -> str:
+        return f"{{session_budget:{session_id}}}:spend"
+
+    async def _get_current_spend(self, cache_key: str) -> float:
+        """Read current accumulated spend for a session."""
+        if (
+            self.internal_usage_cache.dual_cache.redis_cache is not None
+        ):
+            try:
+                result = await self.internal_usage_cache.dual_cache.redis_cache.async_get_cache(
+                    key=cache_key
+                )
+                if result is not None:
+                    return float(result)
+                return 0.0
+            except Exception as e:
+                verbose_proxy_logger.warning(
+                    "MaxBudgetPerSessionHandler: Redis GET failed, "
+                    "falling back to in-memory: %s",
+                    str(e),
+                )
+
+        result = await self.internal_usage_cache.async_get_cache(
+            key=cache_key,
+            litellm_parent_otel_span=None,
+            local_only=True,
+        )
+        if result is not None:
+            return float(result)
+        return 0.0
+
+    async def _increment_spend(self, cache_key: str, amount: float) -> float:
+        """Atomically increment the session spend and return the new value."""
+        if self.increment_script is not None:
+            try:
+                result = await self.increment_script(
+                    keys=[cache_key],
+                    args=[str(amount), self.ttl],
+                )
+                return float(result)
+            except Exception as e:
+                verbose_proxy_logger.warning(
+                    "MaxBudgetPerSessionHandler: Redis INCRBYFLOAT failed, "
+                    "falling back to in-memory: %s",
+                    str(e),
+                )
+
+        return await self._in_memory_increment_spend(cache_key, amount)
+
+    async def _in_memory_increment_spend(
+        self, cache_key: str, amount: float
+    ) -> float:
+        current = await self.internal_usage_cache.async_get_cache(
+            key=cache_key,
+            litellm_parent_otel_span=None,
+            local_only=True,
+        )
+        new_value = (float(current) if current is not None else 0.0) + amount
+        await self.internal_usage_cache.async_set_cache(
+            key=cache_key,
+            value=new_value,
+            ttl=self.ttl,
+            litellm_parent_otel_span=None,
+            local_only=True,
+        )
+        return new_value

litellm/proxy/hooks/max_iterations_limiter.py

@@ -4,7 +4,7 @@ Max Iterations Limiter for LiteLLM Proxy.
 Enforces a per-session cap on the number of LLM calls an agentic loop can make.
 Callers send a `session_id` with each request (via `x-litellm-session-id` header
 or `metadata.session_id`), and this hook counts calls per session. When the count
-exceeds `max_iterations` (configured in key/team metadata), returns 429.
+exceeds `max_iterations` (configured in agent litellm_params or key metadata), returns 429.
 
 Works across multiple proxy instances via DualCache (in-memory + Redis).
 Follows the same pattern as parallel_request_limiter_v3.py.
@@ -52,8 +52,9 @@ class _PROXY_MaxIterationsHandler(CustomLogger):
     Pre-call hook that enforces max_iterations per session.
 
     Configuration:
-        - max_iterations: set in key metadata via /key/generate or /key/update
-          e.g. metadata={"max_iterations": 25}
+        - max_iterations: set in agent litellm_params (preferred)
+          e.g. litellm_params={"max_iterations": 25}
+          Falls back to key metadata max_iterations for backwards compatibility.
         - session_id: sent by caller via x-litellm-session-id header or
           metadata.session_id in request body
 
@@ -93,14 +94,13 @@ class _PROXY_MaxIterationsHandler(CustomLogger):
         Check session iteration count before making the API call.
 
         Extracts session_id from request metadata and max_iterations from
-        key metadata. If the session has exceeded max_iterations, raises 429.
+        agent litellm_params. If the session has exceeded max_iterations, raises 429.
         """
         # Extract session_id from request data
         session_id = self._get_session_id(data)
         if session_id is None:
             return None
 
-        # Extract max_iterations from key metadata
         max_iterations = self._get_max_iterations(user_api_key_dict)
         if max_iterations is None:
             return None
@@ -151,7 +151,22 @@ class _PROXY_MaxIterationsHandler(CustomLogger):
     def _get_max_iterations(
         self, user_api_key_dict: UserAPIKeyAuth
     ) -> Optional[int]:
-        """Extract max_iterations from key metadata."""
+        """Extract max_iterations from agent litellm_params, with fallback to key metadata."""
+        # Try agent litellm_params first
+        agent_id = user_api_key_dict.agent_id
+        if agent_id is not None:
+            from litellm.proxy.agent_endpoints.agent_registry import (
+                global_agent_registry,
+            )
+
+            agent = global_agent_registry.get_agent_by_id(agent_id=agent_id)
+            if agent is not None:
+                litellm_params = agent.litellm_params or {}
+                max_iterations = litellm_params.get("max_iterations")
+                if max_iterations is not None:
+                    return int(max_iterations)
+
+        # Fallback to key metadata for backwards compatibility
         metadata = user_api_key_dict.metadata or {}
         max_iterations = metadata.get("max_iterations")
         if max_iterations is not None:

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -7,18 +7,8 @@ This is currently in development and not yet ready for production.
 import binascii
 import os
 from datetime import datetime
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    Dict,
-    List,
-    Literal,
-    Optional,
-    TypedDict,
-    Union,
-    cast,
-)
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal,
+                    Optional, TypedDict, Union, cast)
 
 from fastapi import HTTPException
 
@@ -175,9 +165,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         """Get or lazy-load the batch rate limiter."""
         if self._batch_rate_limiter is None:
             try:
-                from litellm.proxy.hooks.batch_rate_limiter import (
-                    _PROXY_BatchRateLimiter,
-                )
+                from litellm.proxy.hooks.batch_rate_limiter import \
+                    _PROXY_BatchRateLimiter
 
                 self._batch_rate_limiter = _PROXY_BatchRateLimiter(
                     internal_usage_cache=self.internal_usage_cache,
@@ -679,10 +668,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             requested_model: The model being requested
             descriptors: List of rate limit descriptors to append to
         """
-        from litellm.proxy.auth.auth_utils import (
-            get_key_model_rpm_limit,
-            get_key_model_tpm_limit,
-        )
+        from litellm.proxy.auth.auth_utils import (get_key_model_rpm_limit,
+                                                   get_key_model_tpm_limit)
 
         if not requested_model:
             return
@@ -791,6 +778,92 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         """
         return rpm_limit_type == "dynamic" or tpm_limit_type == "dynamic"
 
+    def _get_agent_from_registry(self, agent_id: str) -> Optional[Any]:
+        """Look up an agent from the in-memory registry by ID."""
+        from litellm.proxy.agent_endpoints.agent_registry import \
+            global_agent_registry
+
+        return global_agent_registry.get_agent_by_id(agent_id=agent_id)
+
+    def _get_resolved_agent_id(
+        self, user_api_key_dict: UserAPIKeyAuth, data: dict
+    ) -> Optional[str]:
+        """
+        Resolve the agent_id from either the API key or request metadata.
+        Key-level agent_id takes precedence over metadata/header-supplied agent_id.
+        """
+        key_agent_id = getattr(user_api_key_dict, "agent_id", None)
+        if key_agent_id:
+            return key_agent_id
+        metadata = data.get("metadata") or {}
+        return metadata.get("agent_id")
+
+    def _get_session_id_from_data(self, data: dict) -> Optional[str]:
+        """Extract session_id from request metadata or litellm_session_id."""
+        session_id = data.get("litellm_session_id")
+        if session_id:
+            return str(session_id)
+        metadata = data.get("metadata") or {}
+        session_id = metadata.get("session_id")
+        if session_id:
+            return str(session_id)
+        litellm_metadata = data.get("litellm_metadata") or {}
+        session_id = litellm_metadata.get("session_id")
+        if session_id:
+            return str(session_id)
+        return None
+
+    def _create_agent_rate_limit_descriptors(
+        self,
+        agent_id: str,
+        data: dict,
+    ) -> List[RateLimitDescriptor]:
+        """
+        Create rate limit descriptors for agent-level and session-level limits.
+
+        Agent-level: caps total RPM/TPM across all sessions for a given agent.
+        Session-level: caps RPM/TPM within a single session (identified by session_id).
+        """
+        descriptors: List[RateLimitDescriptor] = []
+
+        agent = self._get_agent_from_registry(agent_id)
+        if agent is None:
+            return descriptors
+
+        agent_rpm = getattr(agent, "rpm_limit", None)
+        agent_tpm = getattr(agent, "tpm_limit", None)
+        if agent_rpm is not None or agent_tpm is not None:
+            descriptors.append(
+                RateLimitDescriptor(
+                    key="agent",
+                    value=agent_id,
+                    rate_limit={
+                        "requests_per_unit": agent_rpm,
+                        "tokens_per_unit": agent_tpm,
+                        "window_size": self.window_size,
+                    },
+                )
+            )
+
+        session_rpm = getattr(agent, "session_rpm_limit", None)
+        session_tpm = getattr(agent, "session_tpm_limit", None)
+        if session_rpm is not None or session_tpm is not None:
+            session_id = self._get_session_id_from_data(data)
+            if session_id is not None:
+                descriptors.append(
+                    RateLimitDescriptor(
+                        key="agent_session",
+                        value=f"{agent_id}:{session_id}",
+                        rate_limit={
+                            "requests_per_unit": session_rpm,
+                            "tokens_per_unit": session_tpm,
+                            "window_size": self.window_size,
+                        },
+                    )
+                )
+
+        return descriptors
+
     def _create_rate_limit_descriptors(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -802,12 +875,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         """
         Create all rate limit descriptors for the request.
 
-        Returns list of descriptors for API key, user, team, team member, end user, and model-specific limits.
+        Returns list of descriptors for API key, user, team, team member, end user,
+        model-specific, agent, and agent-session limits.
         """
-        from litellm.proxy.auth.auth_utils import (
-            get_team_model_rpm_limit,
-            get_team_model_tpm_limit,
-        )
+        from litellm.proxy.auth.auth_utils import (get_team_model_rpm_limit,
+                                                   get_team_model_tpm_limit)
 
         descriptors = []
 
@@ -956,6 +1028,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                     )
                 )
 
+        # Agent-level and session-level rate limits
+        resolved_agent_id = self._get_resolved_agent_id(user_api_key_dict, data)
+
+        if resolved_agent_id:
+            descriptors.extend(
+                self._create_agent_rate_limit_descriptors(
+                    agent_id=resolved_agent_id,
+                    data=data,
+                )
+            )
+
         return descriptors
 
     async def _check_model_has_recent_failures(
@@ -970,9 +1053,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         Returns True if any deployment has failures in the current minute.
         """
         from litellm.proxy.proxy_server import llm_router
-        from litellm.router_utils.router_callbacks.track_deployment_metrics import (
-            get_deployment_failures_for_current_minute,
-        )
+        from litellm.router_utils.router_callbacks.track_deployment_metrics import \
+            get_deployment_failures_for_current_minute
 
         if llm_router is None:
             return False
@@ -1386,12 +1468,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         """
         Update TPM usage on successful API calls by incrementing counters using pipeline
         """
-        from litellm.litellm_core_utils.core_helpers import (
-            _get_parent_otel_span_from_kwargs,
-        )
-        from litellm.proxy.common_utils.callback_utils import (
-            get_model_group_from_litellm_kwargs,
-        )
+        from litellm.litellm_core_utils.core_helpers import \
+            _get_parent_otel_span_from_kwargs
+        from litellm.proxy.common_utils.callback_utils import \
+            get_model_group_from_litellm_kwargs
         from litellm.types.caching import RedisPipelineIncrementOperation
 
         rate_limit_type = self.get_rate_limit_type()
@@ -1533,6 +1613,32 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                     )
                 )
 
+            # Agent TPM
+            agent_id = standard_logging_metadata.get("agent_id")
+            if agent_id:
+                pipeline_operations.extend(
+                    self._create_pipeline_operations(
+                        key="agent",
+                        value=agent_id,
+                        rate_limit_type="tokens",
+                        total_tokens=total_tokens,
+                    )
+                )
+
+                # Agent Session TPM
+                session_id = standard_logging_metadata.get(
+                    "session_id"
+                ) or standard_logging_metadata.get("trace_id")
+                if session_id:
+                    pipeline_operations.extend(
+                        self._create_pipeline_operations(
+                            key="agent_session",
+                            value=f"{agent_id}:{session_id}",
+                            rate_limit_type="tokens",
+                            total_tokens=total_tokens,
+                        )
+                    )
+
             # Execute all increments in a single pipeline
             if pipeline_operations:
                 await self.async_increment_tokens_with_ttl_preservation(
@@ -1549,9 +1655,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         """
         Decrement max parallel requests counter for the API Key
         """
-        from litellm.litellm_core_utils.core_helpers import (
-            _get_parent_otel_span_from_kwargs,
-        )
+        from litellm.litellm_core_utils.core_helpers import \
+            _get_parent_otel_span_from_kwargs
         from litellm.types.caching import RedisPipelineIncrementOperation
 
         try:

litellm/proxy/litellm_pre_call_utils.py

@@ -690,6 +690,12 @@ class LiteLLMProxyRequestSetup:
             "user_api_key"
         ] = user_api_key_dict.api_key  # this is just the hashed token
 
+        # Key-owned agent_id for spend attribution; keep existing (e.g. from header) if key has none
+        _key_agent_id = getattr(user_api_key_dict, "agent_id", None)
+        _existing_agent_id = data[_metadata_variable_name].get("agent_id")
+        _resolved_agent_id = _key_agent_id or _existing_agent_id
+        data[_metadata_variable_name]["agent_id"] = _resolved_agent_id
+
         data[_metadata_variable_name]["user_api_end_user_max_budget"] = getattr(
             user_api_key_dict, "end_user_max_budget", None
         )

litellm/proxy/management_endpoints/common_utils.py

@@ -41,6 +41,46 @@ def _is_user_team_admin(
     return False
 
 
+async def _is_user_org_admin_for_team(
+    user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_TeamTable
+) -> bool:
+    """
+    Check if user is an org admin for the team's organization.
+
+    Returns True if:
+    - The team belongs to an organization, AND
+    - The user has org_admin role in that organization
+    """
+    if not team_obj.organization_id or not user_api_key_dict.user_id:
+        return False
+
+    from litellm.proxy.auth.auth_checks import get_user_object
+    from litellm.proxy.proxy_server import (
+        prisma_client,
+        proxy_logging_obj,
+        user_api_key_cache,
+    )
+
+    caller_user = await get_user_object(
+        user_id=user_api_key_dict.user_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        user_id_upsert=False,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    if caller_user is None:
+        return False
+
+    for m in caller_user.organization_memberships or []:
+        if (
+            m.organization_id == team_obj.organization_id
+            and m.user_role == LitellmUserRoles.ORG_ADMIN.value
+        ):
+            return True
+
+    return False
+
+
 def _team_member_has_permission(
     user_api_key_dict: UserAPIKeyAuth,
     team_obj: LiteLLM_TeamTable,

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -30,7 +30,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import (
     get_daily_activity,
     get_daily_activity_aggregated,
 )
-from litellm.proxy.auth.auth_checks import get_user_object
+from litellm.proxy.auth.auth_checks import get_team_object, get_user_object
 from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     generate_key_helper_fn,
@@ -1469,6 +1469,72 @@ def _validate_sort_params(
     return order_by
 
 
+async def _authorize_user_list_request(
+    user_api_key_dict: UserAPIKeyAuth,
+    organization_ids: Optional[str],
+    prisma_client: Any,
+    user_api_key_cache: Any,
+    proxy_logging_obj: Any,
+) -> Optional[str]:
+    """
+    Authorize the /user/list request and return the (possibly scoped) organization_ids string.
+
+    - Proxy admins: returns organization_ids unchanged (may be None).
+    - Org admins: returns comma-separated org IDs scoped to their allowed orgs.
+    - Others: raises 403.
+    """
+    if _user_has_admin_view(user_api_key_dict):
+        return organization_ids
+
+    if user_api_key_dict.user_id is None:
+        raise HTTPException(
+            status_code=403,
+            detail={"error": "Only proxy admins and organization admins can list users."},
+        )
+    try:
+        caller_user = await get_user_object(
+            user_id=user_api_key_dict.user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except ValueError:
+        raise HTTPException(
+            status_code=403,
+            detail={"error": "Only proxy admins and organization admins can list users."},
+        )
+    if caller_user is None:
+        raise HTTPException(
+            status_code=403,
+            detail={"error": "Only proxy admins and organization admins can list users."},
+        )
+
+    allowed_org_ids = [
+        m.organization_id
+        for m in (caller_user.organization_memberships or [])
+        if m.user_role == LitellmUserRoles.ORG_ADMIN.value
+    ]
+    if not allowed_org_ids:
+        raise HTTPException(
+            status_code=403,
+            detail={"error": "Only proxy admins and organization admins can list users."},
+        )
+
+    # If client also sent organization_ids, intersect with allowed orgs
+    if organization_ids:
+        requested = set(oid.strip() for oid in organization_ids.split(",") if oid.strip())
+        intersection = list(requested & set(allowed_org_ids))
+        if not intersection:
+            raise HTTPException(
+                status_code=403,
+                detail={"error": "You do not have org_admin access to the requested organization(s)."},
+            )
+        allowed_org_ids = intersection
+
+    return ",".join(allowed_org_ids)
+
+
 @router.get(
     "/user/list",
     tags=["Internal User management"],
@@ -1502,6 +1568,11 @@ async def get_users(
     sort_order: str = fastapi.Query(
         default="asc", description="Sort order ('asc' or 'desc')"
     ),
+    organization_ids: Optional[str] = fastapi.Query(
+        default=None,
+        description="Filter users by organization membership. Comma-separated list of org IDs.",
+    ),
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
 ):
     """
     Get a paginated list of users with filtering and sorting options.
@@ -1530,7 +1601,11 @@ async def get_users(
         sort_order: Optional[str]
             Sort order ('asc' or 'desc')
     """
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm.proxy.proxy_server import (
+        prisma_client,
+        proxy_logging_obj,
+        user_api_key_cache,
+    )
 
     if prisma_client is None:
         raise HTTPException(
@@ -1538,6 +1613,15 @@ async def get_users(
             detail={"error": f"No db connected. prisma client={prisma_client}"},
         )
 
+    # Server-side authorization: proxy admins see all, org admins see only their org(s)
+    organization_ids = await _authorize_user_list_request(
+        user_api_key_dict=user_api_key_dict,
+        organization_ids=organization_ids,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+
     # Calculate skip and take for pagination
     skip = (page - 1) * page_size
 
@@ -1576,6 +1660,13 @@ async def get_users(
             "in": sso_id_list,
         }
 
+    if organization_ids:
+        org_id_list = [oid.strip() for oid in organization_ids.split(",") if oid.strip()]
+        if org_id_list:
+            where_conditions["organization_memberships"] = {
+                "some": {"organization_id": {"in": org_id_list}}
+            }
+
     ## Filter any none fastapi.Query params - e.g. where_conditions: {'user_email': {'contains': Query(None), 'mode': 'insensitive'}, 'teams': {'has': Query(None)}}
     where_conditions = {k: v for k, v in where_conditions.items() if v is not None}
 
@@ -1753,7 +1844,13 @@ async def delete_user(
 
     ## DELETE ASSOCIATED INVITATION LINKS
     await prisma_client.db.litellm_invitationlink.delete_many(
-        where={"user_id": {"in": data.user_ids}}
+        where={
+            "OR": [
+                {"user_id": {"in": data.user_ids}},
+                {"created_by": {"in": data.user_ids}},
+                {"updated_by": {"in": data.user_ids}},
+            ]
+        }
     )
 
     ## DELETE ASSOCIATED ORGANIZATION MEMBERSHIPS
@@ -1820,6 +1917,115 @@ async def add_internal_user_to_organization(
         raise Exception(f"Failed to add user to organization: {str(e)}")
 
 
+async def _resolve_org_filter_for_user_search(
+    user_api_key_dict: UserAPIKeyAuth,
+    team_id: Optional[str],
+    prisma_client: Any,
+    user_api_key_cache: Any,
+    proxy_logging_obj: Any,
+) -> Optional[List[str]]:
+    """
+    Return a list of org IDs to filter by, or ``None`` for no filter.
+
+    Reads the ``scope_user_search_to_org`` UI-setting flag and applies
+    role-based access rules when the flag is ON.
+    """
+    from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import (
+        get_ui_settings_cached,
+    )
+
+    ui_settings = await get_ui_settings_cached()
+    if not ui_settings.get("scope_user_search_to_org", False):
+        return None  # flag OFF — no filtering
+
+    if _user_has_admin_view(user_api_key_dict):
+        return None  # proxy admin — see everything
+
+    # Try to resolve org admin memberships
+    caller_user = None
+    if user_api_key_dict.user_id is not None:
+        try:
+            caller_user = await get_user_object(
+                user_id=user_api_key_dict.user_id,
+                prisma_client=prisma_client,
+                user_api_key_cache=user_api_key_cache,
+                user_id_upsert=False,
+                proxy_logging_obj=proxy_logging_obj,
+            )
+        except ValueError:
+            caller_user = None
+
+    org_admin_org_ids: List[str] = []
+    if caller_user is not None:
+        org_admin_org_ids = [
+            m.organization_id
+            for m in (caller_user.organization_memberships or [])
+            if m.user_role == LitellmUserRoles.ORG_ADMIN.value
+        ]
+
+    if org_admin_org_ids:
+        return org_admin_org_ids
+
+    if team_id is not None:
+        return await _resolve_team_org_filter(
+            user_api_key_dict, team_id, prisma_client,
+            user_api_key_cache, proxy_logging_obj,
+        )
+
+    raise HTTPException(
+        status_code=403,
+        detail={
+            "error": "scope_user_search_to_org is enabled. Only proxy admins, organization admins, or team admins can search users."
+        },
+    )
+
+
+async def _resolve_team_org_filter(
+    user_api_key_dict: UserAPIKeyAuth,
+    team_id: str,
+    prisma_client: Any,
+    user_api_key_cache: Any,
+    proxy_logging_obj: Any,
+) -> List[str]:
+    """Look up the team and return its org as a filter list, or raise 403."""
+    from litellm.proxy.management_endpoints.common_utils import (
+        _is_user_team_admin,
+    )
+
+    try:
+        team_obj = await get_team_object(
+            team_id=team_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except HTTPException:
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "error": f"scope_user_search_to_org is enabled but team '{team_id}' was not found."
+            },
+        )
+
+    if not _is_user_team_admin(user_api_key_dict, team_obj):
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "error": "scope_user_search_to_org is enabled. You must be an admin of this team to search users."
+            },
+        )
+
+    if team_obj.organization_id:
+        return [team_obj.organization_id]
+
+    raise HTTPException(
+        status_code=403,
+        detail={
+            "error": "scope_user_search_to_org is enabled and this team is not part of an organization. Contact your proxy admin to adjust this setting."
+        },
+    )
+
+
 @router.get(
     "/user/filter/ui",
     tags=["Internal User management"],
@@ -1836,6 +2042,10 @@ async def ui_view_users(
     user_email: Optional[str] = fastapi.Query(
         default=None, description="User email in the request parameters"
     ),
+    team_id: Optional[str] = fastapi.Query(
+        default=None,
+        description="Team ID — used when a team admin searches for users to add to their team",
+    ),
     page: int = fastapi.Query(
         default=1, description="Page number for pagination", ge=1
     ),
@@ -1847,19 +2057,15 @@ async def ui_view_users(
     """
     Filter users based on partial match of user_id or email with pagination.
 
-    - Proxy admins: receive all matching users.
-    - Organization admins: receive only users in their own organization(s).
-    - Other roles: access denied (403).
+    Behaviour depends on the ``scope_user_search_to_org`` UI-setting flag
+    (stored in the ``litellm_uisettings`` table):
 
-    Args:
-        user_id (Optional[str]): Partial user ID to search for
-        user_email (Optional[str]): Partial email to search for
-        page (int): Page number for pagination (starts at 1)
-        page_size (int): Number of items per page (max 100)
-        user_api_key_dict (UserAPIKeyAuth): User authentication information
-
-    Returns:
-        List of matching user records (LiteLLM_UserTableFiltered), scoped by org for org admins.
+    * **Flag OFF (default):** any authenticated user can search all users.
+    * **Flag ON:**
+      - Proxy admins see all users.
+      - Org admins see only users in their org(s).
+      - Team admins for an org-bound team see users in that org.
+      - Others receive a 403.
     """
     from litellm.proxy.proxy_server import (
         prisma_client,
@@ -1871,51 +2077,13 @@ async def ui_view_users(
         raise HTTPException(status_code=500, detail={"error": "No db connected"})
 
     try:
-        # Restrict by caller role: proxy admin sees all; org admin sees only their org(s); others 403
-        is_proxy_admin = _user_has_admin_view(user_api_key_dict)
-        if not is_proxy_admin:
-            if user_api_key_dict.user_id is None:
-                raise HTTPException(
-                    status_code=403,
-                    detail={
-                        "error": "Only proxy admins and organization admins can search users."
-                    },
-                )
-            try:
-                caller_user = await get_user_object(
-                    user_id=user_api_key_dict.user_id,
-                    prisma_client=prisma_client,
-                    user_api_key_cache=user_api_key_cache,
-                    user_id_upsert=False,
-                    proxy_logging_obj=proxy_logging_obj,
-                )
-            except ValueError:
-                # get_user_object raises ValueError when user not found (user_id_upsert=False)
-                raise HTTPException(
-                    status_code=403,
-                    detail={
-                        "error": "Only proxy admins and organization admins can search users."
-                    },
-                )
-            if caller_user is None:
-                raise HTTPException(
-                    status_code=403,
-                    detail={
-                        "error": "Only proxy admins and organization admins can search users."
-                    },
-                )
-            org_admin_org_ids = [
-                m.organization_id
-                for m in (caller_user.organization_memberships or [])
-                if m.user_role == LitellmUserRoles.ORG_ADMIN.value
-            ]
-            if not org_admin_org_ids:
-                raise HTTPException(
-                    status_code=403,
-                    detail={
-                        "error": "Only proxy admins and organization admins can search users."
-                    },
-                )
+        org_filter_ids = await _resolve_org_filter_for_user_search(
+            user_api_key_dict=user_api_key_dict,
+            team_id=team_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
 
         # Calculate offset for pagination
         skip = (page - 1) * page_size
@@ -1935,10 +2103,10 @@ async def ui_view_users(
                 "mode": "insensitive",  # Case-insensitive search
             }
 
-        # Org admins: only users in their org(s)
-        if not is_proxy_admin:
+        # Apply org filter when scope_user_search_to_org is ON and caller is not proxy admin
+        if org_filter_ids is not None:
             where_conditions["organization_memberships"] = {
-                "some": {"organization_id": {"in": org_admin_org_ids}}
+                "some": {"organization_id": {"in": org_filter_ids}}
             }
 
         # Query users with pagination and filters

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -4345,8 +4345,6 @@ def _build_key_filter_conditions(
     user_condition: Dict[str, Any] = {}
     if user_id and isinstance(user_id, str):
         user_condition["user_id"] = user_id
-    if team_id and isinstance(team_id, str):
-        user_condition["team_id"] = team_id
     if key_alias and isinstance(key_alias, str):
         user_condition["key_alias"] = key_alias
     if exclude_team_id and isinstance(exclude_team_id, str):
@@ -4414,8 +4412,10 @@ def _build_key_filter_conditions(
     elif len(or_conditions) == 1:
         where.update(or_conditions[0])
 
-    # Apply project_id and access_group_id as global AND filters so they
+    # Apply team_id, project_id and access_group_id as global AND filters so they
     # narrow results across all visibility conditions (own keys, team keys, etc.)
+    if team_id and isinstance(team_id, str):
+        where = {"AND": [where, {"team_id": team_id}]}
     if project_id:
         where = {"AND": [where, {"project_id": project_id}]}
     if access_group_id:

litellm/proxy/management_endpoints/team_endpoints.py

@@ -70,6 +70,7 @@ from litellm.proxy.auth.auth_checks import (
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_endpoints.common_utils import (
+    _is_user_org_admin_for_team,
     _is_user_team_admin,
     _set_object_metadata_field,
     _team_member_has_permission,
@@ -1649,6 +1650,9 @@ async def _validate_team_member_add_permissions(
         and not _is_user_team_admin(
             user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
         )
+        and not await _is_user_org_admin_for_team(
+            user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
+        )
         and not _is_available_team(
             team_id=complete_team_data.team_id,
             user_api_key_dict=user_api_key_dict,
@@ -2121,13 +2125,16 @@ async def team_member_delete(
         )
     existing_team_row = LiteLLM_TeamTable(**_existing_team_row.model_dump())
 
-    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN
+    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN OR ORG ADMIN
 
     if (
         user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
         and not _is_user_team_admin(
             user_api_key_dict=user_api_key_dict, team_obj=existing_team_row
         )
+        and not await _is_user_org_admin_for_team(
+            user_api_key_dict=user_api_key_dict, team_obj=existing_team_row
+        )
     ):
         raise HTTPException(
             status_code=403,
@@ -2280,13 +2287,16 @@ async def team_member_update(
         )
     existing_team_row = LiteLLM_TeamTable(**_existing_team_row.model_dump())
 
-    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN
+    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN OR ORG ADMIN
 
     if (
         user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
         and not _is_user_team_admin(
             user_api_key_dict=user_api_key_dict, team_obj=existing_team_row
         )
+        and not await _is_user_org_admin_for_team(
+            user_api_key_dict=user_api_key_dict, team_obj=existing_team_row
+        )
     ):
         raise HTTPException(
             status_code=403,
@@ -2760,7 +2770,7 @@ async def _persist_deleted_team_records(
         prisma_client=prisma_client,
     )
 
-def validate_membership(
+async def validate_membership(
     user_api_key_dict: UserAPIKeyAuth, team_table: LiteLLM_TeamTable
 ):
     if (
@@ -2795,17 +2805,26 @@ def validate_membership(
                 },
             )
 
-    if user_api_key_dict.user_id not in [
+    # Check direct team membership
+    if user_api_key_dict.user_id in [
         m.user_id for m in team_table.members_with_roles
     ]:
-        raise HTTPException(
-            status_code=403,
-            detail={
-                "error": "User={} not authorized to access this team={}".format(
-                    user_api_key_dict.user_id, team_table.team_id
-                )
-            },
-        )
+        return
+
+    # Check if user is an org admin for the team's organization
+    if await _is_user_org_admin_for_team(
+        user_api_key_dict=user_api_key_dict, team_obj=team_table
+    ):
+        return
+
+    raise HTTPException(
+        status_code=403,
+        detail={
+            "error": "User={} not authorized to access this team={}".format(
+                user_api_key_dict.user_id, team_table.team_id
+            )
+        },
+    )
 
 
 def _unfurl_all_proxy_models(
@@ -2896,7 +2915,7 @@ async def team_info(
                 status_code=status.HTTP_404_NOT_FOUND,
                 detail={"message": f"Team not found, passed team id: {team_id}."},
             )
-        validate_membership(
+        await validate_membership(
             user_api_key_dict=user_api_key_dict,
             team_table=LiteLLM_TeamTable(**team_info.model_dump()),
         )
@@ -3362,6 +3381,101 @@ async def list_team_v2(
     }
 
 
+async def _authorize_and_filter_teams(
+    user_api_key_dict: UserAPIKeyAuth,
+    user_id: Optional[str],
+    prisma_client: Any,
+    user_api_key_cache: Any,
+    proxy_logging_obj: Any,
+) -> list:
+    """
+    Authorize the /team/list request and return filtered teams.
+
+    - Proxy admins: all teams (or filtered by user_id if provided).
+    - Org admins: teams from their orgs + teams they are direct members of.
+    - Own query (user_id matches caller): teams the user is a member of.
+    - Others: 401.
+    """
+    is_proxy_admin = _user_has_admin_view(user_api_key_dict)
+    allowed_org_ids: Optional[List[str]] = None
+
+    if not is_proxy_admin:
+        is_own_query = (
+            user_id is not None
+            and user_api_key_dict.user_id is not None
+            and user_api_key_dict.user_id == user_id
+        )
+
+        # Check if user is an org admin (even for own queries, so they see org teams)
+        if user_api_key_dict.user_id is not None:
+            caller_user = await get_user_object(
+                user_id=user_api_key_dict.user_id,
+                prisma_client=prisma_client,
+                user_api_key_cache=user_api_key_cache,
+                user_id_upsert=False,
+                proxy_logging_obj=proxy_logging_obj,
+            )
+            if caller_user is not None:
+                allowed_org_ids = [
+                    m.organization_id
+                    for m in (caller_user.organization_memberships or [])
+                    if m.user_role == LitellmUserRoles.ORG_ADMIN.value
+                ]
+                if not allowed_org_ids:
+                    allowed_org_ids = None
+
+        if allowed_org_ids is None and not is_own_query:
+            raise HTTPException(
+                status_code=401,
+                detail={
+                    "error": "Only admin users can query all teams/other teams. Your user role={}".format(
+                        user_api_key_dict.user_role
+                    )
+                },
+            )
+
+    if allowed_org_ids is not None:
+        # Org admin: query DB for teams in their orgs
+        org_teams = await prisma_client.db.litellm_teamtable.find_many(
+            where={"organization_id": {"in": allowed_org_ids}},
+            include={"litellm_model_table": True},
+        )
+        if not user_id:
+            return list(org_teams)
+        # Also include teams the user is a direct member of (outside their orgs)
+        seen_team_ids = {team.team_id for team in org_teams}
+        all_teams = list(org_teams)
+        # Prisma doesn't support filtering JSON array fields, so we fetch by membership separately
+        member_teams = await prisma_client.db.litellm_teamtable.find_many(
+            where={"team_id": {"not_in": list(seen_team_ids)}} if seen_team_ids else {},
+            include={"litellm_model_table": True},
+        )
+        for team in member_teams:
+            if team.members_with_roles and any(
+                m.get("user_id") == user_id for m in team.members_with_roles
+            ):
+                all_teams.append(team)
+        return all_teams
+    elif user_id:
+        # Regular user: fetch all and filter by membership (Prisma can't filter JSON arrays)
+        response = await prisma_client.db.litellm_teamtable.find_many(
+            include={"litellm_model_table": True}
+        )
+        return [
+            team
+            for team in response
+            if team.members_with_roles
+            and any(m.get("user_id") == user_id for m in team.members_with_roles)
+        ]
+    else:
+        # Proxy admin: all teams
+        return list(
+            await prisma_client.db.litellm_teamtable.find_many(
+                include={"litellm_model_table": True}
+            )
+        )
+
+
 @router.get(
     "/team/list", tags=["team management"], dependencies=[Depends(user_api_key_auth)]
 )
@@ -3384,19 +3498,11 @@ async def list_team(
     - user_id: str - Optional. If passed will only return teams that the user_id is a member of.
     - organization_id: str - Optional. If passed will only return teams that belong to the organization_id. Pass 'default_organization' to get all teams without organization_id.
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if not allowed_route_check_inside_route(
-        user_api_key_dict=user_api_key_dict, requested_user_id=user_id
-    ):
-        raise HTTPException(
-            status_code=401,
-            detail={
-                "error": "Only admin users can query all teams/other teams. Your user role={}".format(
-                    user_api_key_dict.user_role
-                )
-            },
-        )
+    from litellm.proxy.proxy_server import (
+        prisma_client,
+        proxy_logging_obj,
+        user_api_key_cache,
+    )
 
     if prisma_client is None:
         raise HTTPException(
@@ -3404,27 +3510,14 @@ async def list_team(
             detail={"error": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    response = await prisma_client.db.litellm_teamtable.find_many(
-        include={
-            "litellm_model_table": True,
-        }
+    filtered_response = await _authorize_and_filter_teams(
+        user_api_key_dict=user_api_key_dict,
+        user_id=user_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
     )
 
-    filtered_response = []
-    if user_id:
-        # Get user object to access their teams array
-        for team in response:
-            if team.members_with_roles:
-                for member in team.members_with_roles:
-                    if (
-                        "user_id" in member
-                        and member["user_id"] is not None
-                        and member["user_id"] == user_id
-                    ):
-                        filtered_response.append(team)
-    else:
-        filtered_response = response
-
     _team_ids = [team.team_id for team in filtered_response]
     returned_tm = await get_all_team_memberships(
         prisma_client, _team_ids, user_id=user_id
@@ -3652,12 +3745,15 @@ async def team_model_add(
 
     team_obj = LiteLLM_TeamTable(**team_row.model_dump())
 
-    # Authorization check - only proxy admin or team admin can add models
+    # Authorization check - only proxy admin, team admin, or org admin can add models
     if (
         user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
         and not _is_user_team_admin(
             user_api_key_dict=user_api_key_dict, team_obj=team_obj
         )
+        and not await _is_user_org_admin_for_team(
+            user_api_key_dict=user_api_key_dict, team_obj=team_obj
+        )
     ):
         raise HTTPException(
             status_code=403,
@@ -3720,12 +3816,15 @@ async def team_model_delete(
 
     team_obj = LiteLLM_TeamTable(**team_row.model_dump())
 
-    # Authorization check - only proxy admin or team admin can remove models
+    # Authorization check - only proxy admin, team admin, or org admin can remove models
     if (
         user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
         and not _is_user_team_admin(
             user_api_key_dict=user_api_key_dict, team_obj=team_obj
         )
+        and not await _is_user_org_admin_for_team(
+            user_api_key_dict=user_api_key_dict, team_obj=team_obj
+        )
     ):
         raise HTTPException(
             status_code=403,
@@ -3770,7 +3869,7 @@ async def team_member_permissions(
     if prisma_client is None:
         raise HTTPException(status_code=500, detail={"error": "No db connected"})
 
-    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN
+    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN OR ORG ADMIN
     existing_team_row = await get_team_object(
         team_id=team_id,
         prisma_client=prisma_client,
@@ -3789,6 +3888,9 @@ async def team_member_permissions(
         and not _is_user_team_admin(
             user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
         )
+        and not await _is_user_org_admin_for_team(
+            user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
+        )
         and not _is_available_team(
             team_id=complete_team_data.team_id,
             user_api_key_dict=user_api_key_dict,
@@ -3838,7 +3940,7 @@ async def update_team_member_permissions(
     if prisma_client is None:
         raise HTTPException(status_code=500, detail={"error": "No db connected"})
 
-    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN
+    ## CHECK IF USER IS PROXY ADMIN OR TEAM ADMIN OR ORG ADMIN
     existing_team_row = await get_team_object(
         team_id=data.team_id,
         prisma_client=prisma_client,
@@ -3857,6 +3959,9 @@ async def update_team_member_permissions(
         and not _is_user_team_admin(
             user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
         )
+        and not await _is_user_org_admin_for_team(
+            user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
+        )
         and not _is_available_team(
             team_id=complete_team_data.team_id,
             user_api_key_dict=user_api_key_dict,

litellm/proxy/pass_through_endpoints/streaming_handler.py

@@ -67,6 +67,14 @@ class PassThroughStreamingHandler:
                             )
                             if modified_chunk is not None:
                                 chunk = modified_chunk
+                    elif endpoint_type == EndpointType.ANTHROPIC:
+                        modified_chunk = (
+                            ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection(
+                                chunk, model_name
+                            )
+                        )
+                        if modified_chunk is not None:
+                            chunk = modified_chunk
 
                 yield chunk
 

litellm/proxy/proxy_server.py

@@ -373,9 +373,7 @@ from litellm.proxy.management_endpoints.fallback_management_endpoints import (
 from litellm.proxy.management_endpoints.internal_user_endpoints import (
     router as internal_user_router,
 )
-from litellm.proxy.management_endpoints.internal_user_endpoints import (
-    user_update,
-)
+from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
 from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import (
     router as jwt_key_mapping_router,
 )
@@ -444,9 +442,7 @@ from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_route
 from litellm.proxy.openai_files_endpoints.files_endpoints import (
     router as openai_files_router,
 )
-from litellm.proxy.openai_files_endpoints.files_endpoints import (
-    set_files_config,
-)
+from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     passthrough_endpoint_router,
 )
@@ -545,9 +541,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
     LiteLLM_UpperboundKeyGenerateParams,
 )
 from litellm.types.realtime import RealtimeQueryParams
-from litellm.types.router import (
-    DeploymentTypedDict,
-)
+from litellm.types.router import DeploymentTypedDict
 from litellm.types.router import ModelInfo as RouterModelInfo
 from litellm.types.router import (
     RouterGeneralSettings,
@@ -6682,6 +6676,11 @@ async def chat_completion(  # noqa: PLR0915
             and user_api_key_dict.org_id is not None
         ):
             data["metadata"]["user_api_key_org_id"] = user_api_key_dict.org_id
+        if (
+            hasattr(user_api_key_dict, "agent_id")
+            and user_api_key_dict.agent_id is not None
+        ):
+            data["metadata"]["agent_id"] = user_api_key_dict.agent_id
     base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
     try:
         result = await base_llm_response_processor.base_process_llm_request(
@@ -6851,6 +6850,11 @@ async def completion(  # noqa: PLR0915
                 and user_api_key_dict.org_id is not None
             ):
                 data["metadata"]["user_api_key_org_id"] = user_api_key_dict.org_id
+            if (
+                hasattr(user_api_key_dict, "agent_id")
+                and user_api_key_dict.agent_id is not None
+            ):
+                data["metadata"]["agent_id"] = user_api_key_dict.agent_id
         base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)
         return await base_llm_response_processor.base_process_llm_request(
             request=request,
@@ -7088,6 +7092,11 @@ async def embeddings(  # noqa: PLR0915
                 and user_api_key_dict.org_id is not None
             ):
                 data["metadata"]["user_api_key_org_id"] = user_api_key_dict.org_id
+            if (
+                hasattr(user_api_key_dict, "agent_id")
+                and user_api_key_dict.agent_id is not None
+            ):
+                data["metadata"]["agent_id"] = user_api_key_dict.agent_id
 
         # Use unified request processor (same as chat/completions and responses)
         base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data)

litellm/proxy/schema.prisma

@@ -68,6 +68,11 @@ model LiteLLM_AgentsTable {
   agent_access_groups String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
+  spend      Float    @default(0.0)
+  tpm_limit  Int?
+  rpm_limit  Int?
+  session_tpm_limit Int?
+  session_rpm_limit Int?
   created_at    DateTime               @default(now()) @map("created_at")
   created_by String
   updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -124,6 +124,11 @@ class UISettings(BaseModel):
         description="If true, team admins are exempt from the vector stores disable restriction (only takes effect when disable_vector_stores_for_internal_users is true).",
     )
 
+    scope_user_search_to_org: bool = Field(
+        default=False,
+        description="If enabled, the user search endpoint (/user/filter/ui) restricts results by organization. When off, any authenticated user can search all users.",
+    )
+
 
 class UISettingsResponse(SettingsResponse):
     """Response model for UI settings"""
@@ -143,6 +148,7 @@ ALLOWED_UI_SETTINGS_FIELDS = {
     "allow_agents_for_team_admins",
     "disable_vector_stores_for_internal_users",
     "allow_vector_stores_for_team_admins",
+    "scope_user_search_to_org",
 }
 
 # Flags that must be synced from the persisted UISettings into
@@ -974,6 +980,49 @@ async def get_in_product_nudges():
     return InProductNudgeResponse(is_claude_code_enabled=False)
 
 
+UI_SETTINGS_CACHE_KEY = "ui_settings:settings_dict"
+UI_SETTINGS_CACHE_TTL = 600  # 10 minutes
+
+
+async def get_ui_settings_cached() -> Dict[str, Any]:
+    """
+    Return the persisted UI settings dict, using DualCache for reads.
+
+    Cache hit  → return cached dict immediately.
+    Cache miss → read from DB, populate cache, return dict.
+    """
+    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
+
+    # 1. Try cache
+    cached = await user_api_key_cache.async_get_cache(key=UI_SETTINGS_CACHE_KEY)
+    if cached is not None and isinstance(cached, dict):
+        return cached
+
+    # 2. Fallback to DB
+    if prisma_client is None:
+        return {}
+
+    db_record = await prisma_client.db.litellm_uisettings.find_unique(
+        where={"id": "ui_settings"}
+    )
+    ui_settings: Dict[str, Any] = {}
+    if db_record and db_record.ui_settings:
+        raw = db_record.ui_settings
+        ui_settings = json.loads(raw) if isinstance(raw, str) else dict(raw)
+
+    # Sanitize
+    ui_settings = {
+        k: v for k, v in ui_settings.items() if k in ALLOWED_UI_SETTINGS_FIELDS
+    }
+
+    # 3. Populate cache with TTL
+    await user_api_key_cache.async_set_cache(
+        key=UI_SETTINGS_CACHE_KEY, value=ui_settings, ttl=UI_SETTINGS_CACHE_TTL
+    )
+
+    return ui_settings
+
+
 @router.get(
     "/get/ui_settings",
     tags=["UI Settings"],
@@ -1018,6 +1067,13 @@ async def get_ui_settings():
 
         general_settings.update(_flags_to_sync)
 
+    # Refresh DualCache so other code paths (e.g. /user/filter/ui) see fresh values
+    from litellm.proxy.proxy_server import user_api_key_cache
+
+    await user_api_key_cache.async_set_cache(
+        key=UI_SETTINGS_CACHE_KEY, value=ui_settings, ttl=UI_SETTINGS_CACHE_TTL
+    )
+
     # Build config-like object for schema helper
     config: Dict[str, Any] = {"litellm_settings": {"ui_settings": ui_settings}}
 
@@ -1102,6 +1158,16 @@ async def update_ui_settings(
 
         general_settings.update(_flags_to_sync)
 
+    # Invalidate + set DualCache so subsequent reads see the new values immediately
+    from litellm.proxy.proxy_server import user_api_key_cache
+
+    sanitized = {
+        k: v for k, v in ui_settings.items() if k in ALLOWED_UI_SETTINGS_FIELDS
+    }
+    await user_api_key_cache.async_set_cache(
+        key=UI_SETTINGS_CACHE_KEY, value=sanitized, ttl=UI_SETTINGS_CACHE_TTL
+    )
+
     return {
         "message": "UI settings updated successfully",
         "status": "success",

litellm/proxy/utils.py

@@ -325,7 +325,7 @@ class ProxyLogging:
             if email_logger_class is not None:
                 # All email logger classes now accept internal_usage_cache
                 self.email_logging_instance = email_logger_class(
-                    internal_usage_cache=self.internal_usage_cache.dual_cache,
+                    internal_usage_cache=self.internal_usage_cache.dual_cache,  # type: ignore[call-arg]
                 )
         self.premium_user = premium_user
         self.service_logging_obj = ServiceLogging()
@@ -5279,7 +5279,7 @@ async def get_available_models_for_user(
             user_api_key_cache=user_api_key_cache,
             proxy_logging_obj=proxy_logging_obj,
         )
-        validate_membership(user_api_key_dict=user_api_key_dict, team_table=team_object)
+        await validate_membership(user_api_key_dict=user_api_key_dict, team_table=team_object)
         team_models = team_object.models
 
     team_models = get_team_models(

litellm/router.py

@@ -164,11 +164,7 @@ from litellm.types.utils import (
 )
 from litellm.types.utils import ModelInfo
 from litellm.types.utils import ModelInfo as ModelMapInfo
-from litellm.types.utils import (
-    ModelResponseStream,
-    StandardLoggingPayload,
-    Usage,
-)
+from litellm.types.utils import ModelResponseStream, StandardLoggingPayload, Usage
 from litellm.utils import (
     CustomStreamWrapper,
     EmbeddingResponse,

litellm/types/agents.py

@@ -179,6 +179,10 @@ class AgentConfig(TypedDict, total=False):
     agent_card_params: Required[AgentCard]
     litellm_params: Dict[str, Any]  # allow for any future litellm params
     object_permission: AgentObjectPermission
+    tpm_limit: Optional[int]
+    rpm_limit: Optional[int]
+    session_tpm_limit: Optional[int]
+    session_rpm_limit: Optional[int]
     static_headers: Optional[Dict[str, str]]
     extra_headers: Optional[List[str]]
 
@@ -188,6 +192,10 @@ class PatchAgentRequest(TypedDict, total=False):
     agent_card_params: AgentCard
     litellm_params: Dict[str, Any]
     object_permission: AgentObjectPermission
+    tpm_limit: Optional[int]
+    rpm_limit: Optional[int]
+    session_tpm_limit: Optional[int]
+    session_rpm_limit: Optional[int]
     static_headers: Optional[Dict[str, str]]
     extra_headers: Optional[List[str]]
 
@@ -201,6 +209,11 @@ class AgentResponse(BaseModel):
     litellm_params: Optional[Dict[str, Any]] = None
     agent_card_params: Dict[str, Any]
     object_permission: Optional[Dict[str, Any]] = None
+    spend: Optional[float] = None
+    tpm_limit: Optional[int] = None
+    rpm_limit: Optional[int] = None
+    session_tpm_limit: Optional[int] = None
+    session_rpm_limit: Optional[int] = None
     static_headers: Optional[Dict[str, str]] = None
     extra_headers: Optional[List[str]] = None
     created_at: Optional[datetime] = None

litellm/types/llms/custom_http.py

@@ -25,6 +25,7 @@ class httpxSpecialProvider(str, Enum):
     MCP = "mcp"
     RAG = "rag"
     A2AProvider = "a2a_provider"
+    AgentHealthCheck = "agent_health_check"
     A2A = "a2a"
     PromptManagement = "prompt_management"
     UI = "ui"

litellm/types/router.py

@@ -8,7 +8,7 @@ from dataclasses import dataclass
 from typing import Any, Dict, List, Literal, Optional, Tuple, Union, get_type_hints
 
 import httpx
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, model_validator
 from typing_extensions import Required, TypedDict
 
 from litellm._uuid import uuid
@@ -16,7 +16,6 @@ from litellm._uuid import uuid
 from .completion import CompletionRequest
 from .embedding import EmbeddingRequest
 from .llms.openai import OpenAIFileObject
-from .llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
 from .search import SearchProvider
 from .utils import CustomPricingLiteLLMParams, ModelResponse
 
@@ -162,6 +161,9 @@ class CredentialLiteLLMParams(BaseModel):
     watsonx_region_name: Optional[str] = None
 
 
+_RESERVED_INIT_KEYS = frozenset({"self", "params", "__class__"})
+
+
 class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     """
     LiteLLM Params without 'model' arg (used across completion / assistants api)
@@ -215,76 +217,21 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     vector_store_id: Optional[str] = None
     milvus_text_field: Optional[str] = None
 
-    def __init__(
-        self,
-        custom_llm_provider: Optional[str] = None,
-        max_retries: Optional[Union[int, str]] = None,
-        tpm: Optional[int] = None,
-        rpm: Optional[int] = None,
-        api_key: Optional[str] = None,
-        api_base: Optional[str] = None,
-        api_version: Optional[str] = None,
-        timeout: Optional[Union[float, str]] = None,  # if str, pass in as os.environ/
-        stream_timeout: Optional[Union[float, str]] = (
-            None  # timeout when making stream=True calls, if str, pass in as os.environ/
-        ),
-        organization: Optional[str] = None,  # for openai orgs
-        ## LOGGING PARAMS ##
-        litellm_trace_id: Optional[str] = None,
-        ## UNIFIED PROJECT/REGION ##
-        region_name: Optional[str] = None,
-        ## VERTEX AI ##
-        vertex_project: Optional[str] = None,
-        vertex_location: Optional[str] = None,
-        vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES] = None,
-        ## AWS BEDROCK / SAGEMAKER ##
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        aws_region_name: Optional[str] = None,
-        ## IBM WATSONX ##
-        watsonx_region_name: Optional[str] = None,
-        input_cost_per_token: Optional[float] = None,
-        output_cost_per_token: Optional[float] = None,
-        input_cost_per_second: Optional[float] = None,
-        output_cost_per_second: Optional[float] = None,
-        max_file_size_mb: Optional[float] = None,
-        # Deployment budgets
-        max_budget: Optional[float] = None,
-        budget_duration: Optional[str] = None,
-        # Pass through params
-        use_in_pass_through: Optional[bool] = False,
-        # Dynamic param to force using litellm proxy
-        use_litellm_proxy: Optional[bool] = False,
-        # This will merge the reasoning content in the choices
-        merge_reasoning_content_in_choices: Optional[bool] = False,
-        model_info: Optional[Dict] = None,
-        mock_response: Optional[Union[str, ModelResponse, Exception, Any]] = None,
-        # auto-router params
-        auto_router_config_path: Optional[str] = None,
-        auto_router_config: Optional[str] = None,
-        auto_router_default_model: Optional[str] = None,
-        auto_router_embedding_model: Optional[str] = None,
-        # complexity-router params
-        complexity_router_config: Optional[Dict] = None,
-        complexity_router_default_model: Optional[str] = None,
-        # Batch/File API Params
-        s3_bucket_name: Optional[str] = None,
-        s3_encryption_key_id: Optional[str] = None,
-        gcs_bucket_name: Optional[str] = None,
-        **params,
-    ):
-        args = locals()
-        args.pop("max_retries", None)
-        args.pop("self", None)
-        args.pop("params", None)
-        args.pop("__class__", None)
-        if max_retries is not None and isinstance(max_retries, str):
-            max_retries = int(max_retries)  # cast to int
-        # We need to keep max_retries in args since it's a parameter of GenericLiteLLMParams
-        args[
-            "max_retries"
-        ] = max_retries  # Put max_retries back in args after popping it
-        super().__init__(**args, **params)
+    @model_validator(mode="before")
+    @classmethod
+    def preprocess_input_data(cls, data: Any) -> Any:
+        """
+        Pre-process input data before validation:
+        1. Filter out reserved Python keywords ('self', 'params', '__class__') to prevent
+           'got multiple values for argument' errors when user data contains these keys.
+        2. Convert max_retries from string to int if needed.
+        """
+        if isinstance(data, dict):
+            filtered = {k: v for k, v in data.items() if k not in _RESERVED_INIT_KEYS}
+            if "max_retries" in filtered and isinstance(filtered["max_retries"], str):
+                filtered["max_retries"] = int(filtered["max_retries"])
+            return filtered
+        return data
 
     def __contains__(self, key):
         # Define custom behavior for the 'in' operator
@@ -311,46 +258,6 @@ class LiteLLM_Params(GenericLiteLLMParams):
     model: str
     model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
 
-    def __init__(
-        self,
-        model: str,
-        custom_llm_provider: Optional[str] = None,
-        max_retries: Optional[Union[int, str]] = None,
-        tpm: Optional[int] = None,
-        rpm: Optional[int] = None,
-        api_key: Optional[str] = None,
-        api_base: Optional[str] = None,
-        api_version: Optional[str] = None,
-        timeout: Optional[Union[float, str]] = None,  # if str, pass in as os.environ/
-        stream_timeout: Optional[Union[float, str]] = (
-            None  # timeout when making stream=True calls, if str, pass in as os.environ/
-        ),
-        organization: Optional[str] = None,  # for openai orgs
-        ## VERTEX AI ##
-        vertex_project: Optional[str] = None,
-        vertex_location: Optional[str] = None,
-        ## AWS BEDROCK / SAGEMAKER ##
-        aws_access_key_id: Optional[str] = None,
-        aws_secret_access_key: Optional[str] = None,
-        aws_region_name: Optional[str] = None,
-        # OpenAI / Azure Whisper
-        # set a max-size of file that can be passed to litellm proxy
-        max_file_size_mb: Optional[float] = None,
-        # will use deployment on pass-through endpoints if True
-        use_in_pass_through: Optional[bool] = False,
-        use_litellm_proxy: Optional[bool] = False,
-        **params,
-    ):
-        args = locals()
-        args.pop("max_retries", None)
-        args.pop("self", None)
-        args.pop("params", None)
-        args.pop("__class__", None)
-        if max_retries is not None and isinstance(max_retries, str):
-            max_retries = int(max_retries)  # cast to int
-        args["max_retries"] = max_retries
-        super().__init__(**{**args, **params})
-
     def __contains__(self, key):
         # Define custom behavior for the 'in' operator
         return hasattr(self, key)

litellm/types/utils.py

@@ -3249,6 +3249,7 @@ class SearchProviders(str, Enum):
     LINKUP = "linkup"
     DUCKDUCKGO = "duckduckgo"
     SEARCHAPI = "searchapi"
+    SERPER = "serper"
 
 
 # Create a set of all search provider values for quick lookup

litellm/utils.py

@@ -8884,6 +8884,7 @@ class ProviderConfigManager:
         from litellm.llms.perplexity.search.transformation import PerplexitySearchConfig
         from litellm.llms.searchapi.search.transformation import SearchAPIConfig
         from litellm.llms.searxng.search.transformation import SearXNGSearchConfig
+        from litellm.llms.serper.search.transformation import SerperSearchConfig
         from litellm.llms.tavily.search.transformation import TavilySearchConfig
 
         PROVIDER_TO_CONFIG_MAP = {
@@ -8899,6 +8900,7 @@ class ProviderConfigManager:
             SearchProviders.LINKUP: LinkupSearchConfig,
             SearchProviders.DUCKDUCKGO: DuckDuckGoSearchConfig,
             SearchProviders.SEARCHAPI: SearchAPIConfig,
+            SearchProviders.SERPER: SerperSearchConfig,
         }
         config_class = PROVIDER_TO_CONFIG_MAP.get(provider, None)
         if config_class is None:

model_prices_and_context_window.json

@@ -4207,6 +4207,41 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "azure/gpt-5.3-chat": {
+        "cache_read_input_token_cost": 1.75e-07,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "input_cost_per_token": 1.75e-06,
+        "input_cost_per_token_priority": 3.5e-06,
+        "litellm_provider": "azure",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "output_cost_per_token_priority": 2.8e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_service_tier": true,
+        "supports_vision": true
+    },
     "azure/gpt-5.3-codex": {
         "cache_read_input_token_cost": 1.75e-07,
         "input_cost_per_token": 1.75e-06,
@@ -4299,6 +4334,160 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "azure/gpt-5.4": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost_priority": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
+        "input_cost_per_token": 2.5e-06,
+        "input_cost_per_token_above_272k_tokens": 5e-06,
+        "input_cost_per_token_priority": 5e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 1e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "output_cost_per_token_above_272k_tokens": 2.25e-05,
+        "output_cost_per_token_priority": 3e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_service_tier": true,
+        "supports_vision": true
+    },
+    "azure/gpt-5.4-2026-03-05": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost_priority": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1e-06,
+        "input_cost_per_token": 2.5e-06,
+        "input_cost_per_token_above_272k_tokens": 5e-06,
+        "input_cost_per_token_priority": 5e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 1e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "output_cost_per_token_above_272k_tokens": 2.25e-05,
+        "output_cost_per_token_priority": 3e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 4.5e-05,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_service_tier": true,
+        "supports_vision": true
+    },
+    "azure/gpt-5.4-pro": {
+        "cache_read_input_token_cost": 3e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 6e-06,
+        "input_cost_per_token": 3e-05,
+        "input_cost_per_token_above_272k_tokens": 6e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 0.00018,
+        "output_cost_per_token_above_272k_tokens": 0.00027,
+        "supported_endpoints": [
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "azure/gpt-5.4-pro-2026-03-05": {
+        "cache_read_input_token_cost": 3e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 6e-06,
+        "input_cost_per_token": 3e-05,
+        "input_cost_per_token_above_272k_tokens": 6e-05,
+        "litellm_provider": "azure",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "output_cost_per_token": 0.00018,
+        "output_cost_per_token_above_272k_tokens": 0.00027,
+        "supported_endpoints": [
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "azure/gpt-image-1": {
         "cache_read_input_image_token_cost": 2.5e-06,
         "cache_read_input_token_cost": 1.25e-06,
@@ -12090,6 +12279,14 @@
             "notes": "SearXNG is an open-source metasearch engine. Free to use when self-hosted or using public instances."
         }
     },
+    "serper/search": {
+        "input_cost_per_query": 0.001,
+        "litellm_provider": "serper",
+        "mode": "search",
+        "metadata": {
+            "notes": "Serper Google Search API. Pricing: $1.00/1k queries (Starter), $0.75/1k (Standard), $0.50/1k (Scale), $0.30/1k (Ultimate)."
+        }
+    },
     "elevenlabs/scribe_v1": {
         "input_cost_per_second": 6.11e-05,
         "litellm_provider": "elevenlabs",
@@ -16799,6 +16996,42 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "gemini/gemini-3.1-flash-image-preview": {
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.045,
+        "output_cost_per_image_token": 6e-05,
+        "output_cost_per_image_token_batches": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "rpm": 1000,
+        "tpm": 4000000,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-image-preview",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": false,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "gemini/deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,
@@ -21083,7 +21316,7 @@
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
-        "mode": "chat",
+        "mode": "responses",
         "output_cost_per_token": 0.00018,
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
@@ -21091,9 +21324,8 @@
         "output_cost_per_token_priority": 0.00027,
         "output_cost_per_token_above_272k_tokens_priority": 0.000405,
         "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/batch",
-            "/v1/responses"
+            "/v1/responses",
+            "/v1/batch"
         ],
         "supported_modalities": [
             "text",
@@ -21132,7 +21364,7 @@
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
-        "mode": "chat",
+        "mode": "responses",
         "output_cost_per_token": 0.00018,
         "output_cost_per_token_above_272k_tokens": 0.00027,
         "output_cost_per_token_flex": 9e-05,
@@ -21140,9 +21372,8 @@
         "output_cost_per_token_priority": 0.00027,
         "output_cost_per_token_above_272k_tokens_priority": 0.000405,
         "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/batch",
-            "/v1/responses"
+            "/v1/responses",
+            "/v1/batch"
         ],
         "supported_modalities": [
             "text",

package.json

@@ -12,8 +12,8 @@
   },
   "overrides": {
     "glob": ">=11.1.0",
-    "tar": ">=7.5.8",
-    "minimatch": ">=10.2.1",
+    "tar": ">=7.5.10",
+    "minimatch": ">=10.2.4",
     "diff": ">=8.0.3",
     "@isaacs/brace-expansion": ">=5.0.1",
     "@babel/traverse": ">=7.23.2",

poetry.lock

@@ -3222,15 +3222,15 @@ files = [
 
 [[package]]
 name = "litellm-proxy-extras"
-version = "0.4.51"
+version = "0.4.53"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 optional = true
 python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
 groups = ["main"]
 markers = "extra == \"proxy\""
 files = [
-    {file = "litellm_proxy_extras-0.4.51-py3-none-any.whl", hash = "sha256:4ca8c1e131fc5c3cb0a47ae4d6971c784c211f5b83021d1c7fdeb9831c8f5070"},
-    {file = "litellm_proxy_extras-0.4.51.tar.gz", hash = "sha256:785738cd647c5b4da9fb78efa5cce1c7189c176b7feef971cbab0982a72f8fc0"},
+    {file = "litellm_proxy_extras-0.4.53-py3-none-any.whl", hash = "sha256:9224c667144774b6119e4de9b4b2d52fafc58442e6db317785c43b2d833665d6"},
+    {file = "litellm_proxy_extras-0.4.53.tar.gz", hash = "sha256:22c53fa8890d93d4a0d24171726e4e2bba8be6fef4838317cb74284fa9d27f70"},
 ]
 
 [[package]]
@@ -8002,4 +8002,4 @@ utils = ["numpydoc"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.9,<4.0"
-content-hash = "87adea65389e69a97651f6b100bf1566249d86e308b327b018a356e41ab6b116"
+content-hash = "3036cfcdc06fb4293e248a2edd9c32a7afe6846920167527e247b2aefd74cfa6"

provider_endpoints_support.json

@@ -2061,6 +2061,13 @@
         "search": true
       }
     },
+    "serper": {
+      "display_name": "Serper (`serper`)",
+      "url": "https://docs.litellm.ai/docs/search/serper",
+      "endpoints": {
+        "search": true
+      }
+    },
     "triton": {
       "display_name": "Triton (`triton`)",
       "url": "https://docs.litellm.ai/docs/providers/triton-inference-server",

pyproject.toml

@@ -47,7 +47,7 @@ fastapi-sso = { version = "^0.16.0", optional = true }
 PyJWT = { version = "^2.10.1", optional = true, python = ">=3.9" }
 python-multipart = { version = ">=0.0.20", optional = true}
 cryptography = {version = "*", optional = true}
-prisma = {version = "0.11.0", optional = true}
+prisma = {version = "^0.11.0", optional = true}
 azure-identity = {version = "^1.15.0", optional = true, python = ">=3.9"}
 azure-keyvault-secrets = {version = "^4.8.0", optional = true}
 azure-storage-blob = {version="^12.25.1", optional=true}
@@ -57,13 +57,13 @@ google-cloud-aiplatform = {version = ">=1.38.0", optional = true}
 resend = {version = ">=0.8.0", optional = true}
 pynacl = {version = "^1.5.0", optional = true}
 websockets = {version = "^15.0.1", optional = true}
-boto3 = { version = "1.40.76", optional = true }
+boto3 = { version = "^1.40.76", optional = true }
 redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
 mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"}
 a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"}
-litellm-proxy-extras = {version = "0.4.52", optional = true}
-rich = {version = "13.7.1", optional = true}
-litellm-enterprise = {version = "0.1.33", optional = true}
+litellm-proxy-extras = {version = "^0.4.53", optional = true}
+rich = {version = "^13.7.1", optional = true}
+litellm-enterprise = {version = "^0.1.33", optional = true}
 diskcache = {version = "^5.6.1", optional = true}
 polars = {version = "^1.31.0", optional = true, python = ">=3.10"}
 semantic-router = {version = ">=0.1.12", optional = true, python = ">=3.9,<3.14"}

requirements.txt

@@ -57,7 +57,7 @@ grpcio>=1.75.0; python_version >= "3.14"
 sentry_sdk==2.21.0 # for sentry error handling
 detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
 tzdata==2025.1 # IANA time zone database
-litellm-proxy-extras==0.4.52 # for proxy extras - e.g. prisma migrations
+litellm-proxy-extras==0.4.53 # for proxy extras - e.g. prisma migrations
 llm-sandbox==0.3.31 # for skill execution in sandbox
 ### LITELLM PACKAGE DEPENDENCIES
 python-dotenv==1.0.1 # for env
@@ -75,9 +75,9 @@ jsonschema>=4.23.0,<5.0.0 # validating json schema - aligned with openapi-core +
 websockets==15.0.1 # for realtime API
 soundfile==0.12.1 # for audio file processing
 openapi-core==0.21.0 # for OpenAPI compliance tests
-pypdf>=6.6.2 # for PDF text extraction in RAG ingestion
+pypdf>=6.7.3 # for PDF text extraction in RAG ingestion (CVE-2026-27888)
 
 ########################
 # LITELLM ENTERPRISE DEPENDENCIES
 ########################
-litellm-enterprise==0.1.33
+litellm-enterprise==0.1.34

schema.prisma

@@ -68,6 +68,11 @@ model LiteLLM_AgentsTable {
   agent_access_groups String[]       @default([])
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
+  spend      Float    @default(0.0)
+  tpm_limit  Int?
+  rpm_limit  Int?
+  session_tpm_limit Int?
+  session_rpm_limit Int?
   created_at    DateTime               @default(now()) @map("created_at")
   created_by String
   updated_at    DateTime               @default(now()) @updatedAt @map("updated_at")

tests/code_coverage_tests/enforce_llms_folder_style.py

@@ -18,6 +18,7 @@ SEARCH_PROVIDERS = [
     "linkup",
     "duckduckgo",
     "searchapi",
+    "serper",
 ]
 
 ALLOWED_FILES_IN_LLMS_FOLDER = [

tests/image_gen_tests/test_image_variation.py

@@ -27,62 +27,67 @@ import tempfile
 from base_image_generation_test import BaseImageGenTest
 import logging
 from litellm._logging import verbose_logger
-import requests
 from io import BytesIO
+from PIL import Image as PILImage
 
 verbose_logger.setLevel(logging.DEBUG)
 
 
 @pytest.fixture
 def image_url():
-    # URL of the image
-    image_url = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
-
-    # Fetch the image from the URL
-    response = requests.get(image_url)
-    print(response)
-    response.raise_for_status()  # Ensure the request was successful
-
-    # Load the image into a file-like object
-    image_file = BytesIO(response.content)
+    # DALL-E 2 image variations require a square PNG (less than 4MB)
+    # Generate a 1024x1024 square PNG programmatically to avoid network dependency
+    # and the non-square aspect ratio of the old LiteLLM logo URL
+    img = PILImage.new("RGBA", (1024, 1024), color=(128, 128, 128, 255))
+    image_file = BytesIO()
+    img.save(image_file, format="PNG")
+    image_file.seek(0)
+    # openai>=2.24.0 requires BytesIO to have .name for MIME type detection in multipart uploads
+    image_file.name = "litellm_logo.png"
 
     return image_file
 
 
-def test_openai_image_variation_openai_sdk(image_url):
-    from openai import OpenAI
-
-    client = OpenAI()
-    response = client.images.create_variation(image=image_url, n=2, size="1024x1024")
-    print(response)
+# Commented out: OpenAI /images/variations endpoint deprecated (DALL-E 2 shutdown May 12, 2026)
+# def test_openai_image_variation_openai_sdk(image_url):
+#     from openai import OpenAI
+#
+#     client = OpenAI()
+#     response = client.images.create_variation(image=image_url, n=2, size="1024x1024")
+#     print(response)
+#
+#
+# @pytest.mark.parametrize("sync_mode", [True, False])
+# @pytest.mark.asyncio
+# async def test_openai_image_variation_litellm_sdk(image_url, sync_mode):
+#     from litellm import image_variation, aimage_variation
+#
+#     if sync_mode:
+#         image_variation(image=image_url, n=2, size="1024x1024")
+#     else:
+#         await aimage_variation(image=image_url, n=2, size="1024x1024")
+#
+#
+# def test_topaz_image_variation(image_url):
+#     from litellm import image_variation, aimage_variation
+#     from litellm.llms.custom_httpx.http_handler import HTTPHandler
+#     from unittest.mock import patch
+#
+#     client = HTTPHandler()
+#     with patch.object(client, "post") as mock_post:
+#         try:
+#             image_variation(
+#                 model="topaz/Standard V2",
+#                 image=image_url,
+#                 n=2,
+#                 size="1024x1024",
+#                 client=client,
+#             )
+#         except Exception as e:
+#             print(e)
+#         mock_post.assert_called_once()
 
 
-@pytest.mark.parametrize("sync_mode", [True, False])
-@pytest.mark.asyncio
-async def test_openai_image_variation_litellm_sdk(image_url, sync_mode):
-    from litellm import image_variation, aimage_variation
-
-    if sync_mode:
-        image_variation(image=image_url, n=2, size="1024x1024")
-    else:
-        await aimage_variation(image=image_url, n=2, size="1024x1024")
-
-
-def test_topaz_image_variation(image_url):
-    from litellm import image_variation, aimage_variation
-    from litellm.llms.custom_httpx.http_handler import HTTPHandler
-    from unittest.mock import patch
-
-    client = HTTPHandler()
-    with patch.object(client, "post") as mock_post:
-        try:
-            image_variation(
-                model="topaz/Standard V2",
-                image=image_url,
-                n=2,
-                size="1024x1024",
-                client=client,
-            )
-        except Exception as e:
-            print(e)
-        mock_post.assert_called_once()
+def test_image_variation_placeholder():
+    """Placeholder: variation tests commented out - OpenAI /images/variations deprecated (DALL-E 2 shutdown May 12, 2026)."""
+    pass

tests/llm_translation/base_llm_unit_tests.py

@@ -868,8 +868,9 @@ class BaseLLMChatTest(ABC):
         base_completion_call_args = self.get_base_completion_call_args()
         if not supports_vision(base_completion_call_args["model"], None):
             pytest.skip("Model does not support image input")
-        elif "http://" in image_url and "fireworks_ai" in base_completion_call_args.get(
-            "model"
+        elif "http://" in image_url and (
+            "fireworks_ai" in base_completion_call_args.get("model", "")
+            or "mistral" in base_completion_call_args.get("model", "")
         ):
             pytest.skip("Model does not support http:// input")
 

tests/llm_translation/test_openrouter.py

@@ -30,7 +30,7 @@ def test_completion_openrouter_image_generation():
     assert (
         resp.choices[0]
         .message.images[0]["image_url"]["url"]
-        .startswith("data:image/png;base64,")
+        .startswith("data:image/")
     )
 
 

tests/llm_translation/test_skills_api.py

@@ -23,27 +23,42 @@ from litellm.types.llms.anthropic_skills import (
 
 
 @contextmanager
-def create_skill_zip(skill_name: str):
+def create_skill_zip(skill_name: str, unique_suffix: Optional[str] = None):
     """
     Helper context manager to create a zip file for a skill.
-    
+
     Args:
         skill_name: Name of the skill directory in test_skills_data/
-        
+        unique_suffix: Optional suffix to make the skill name unique in the zip.
+                       When provided, the SKILL.md frontmatter name is rewritten
+                       to avoid duplicate-name conflicts on the API side.
+
     Yields:
         File handle to the zip file
-        
+
     The zip file is automatically cleaned up after use.
     """
+    import time
+
     test_dir = Path(__file__).parent / "test_skills_data"
     skill_dir = test_dir / skill_name
-    
+
     # Create a zip file containing the skill directory
     zip_path = test_dir / f"{skill_name}.zip"
-    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zip_file:
-        zip_file.write(skill_dir, arcname=skill_name)
-        zip_file.write(skill_dir / "SKILL.md", arcname=f"{skill_name}/SKILL.md")
-    
+    with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+        zf.write(skill_dir, arcname=skill_name)
+
+        if unique_suffix is not None:
+            # Rewrite SKILL.md with a unique name to avoid API conflicts
+            skill_md = (skill_dir / "SKILL.md").read_text()
+            skill_md = skill_md.replace(
+                f"name: {skill_name}",
+                f"name: {skill_name}-{unique_suffix}",
+            )
+            zf.writestr(f"{skill_name}/SKILL.md", skill_md)
+        else:
+            zf.write(skill_dir / "SKILL.md", arcname=f"{skill_name}/SKILL.md")
+
     try:
         with open(zip_path, "rb") as f:
             yield f
@@ -77,13 +92,13 @@ class BaseSkillsAPITest(ABC):
     def test_create_skill(self):
         """
         Test creating a skill.
-        
+
         Note: This test creates a skill but does not clean it up,
         as we want to verify it was created successfully.
         The test_delete_skill test will handle cleanup.
         """
         import time
-        
+
         custom_llm_provider = self.get_custom_llm_provider()
         api_key = self.get_api_key()
         api_base = self.get_api_base()
@@ -96,12 +111,14 @@ class BaseSkillsAPITest(ABC):
 
         # Use helper to create skill zip
         skill_name = "test-skill-litellm"
-        
-        # Use unique title to avoid conflicts with previous test runs
-        unique_title = f"Test Skill {int(time.time())}"
-        
+
+        # Use unique title and unique skill name to avoid conflicts
+        # with previous test runs (skills are never cleaned up in CI)
+        ts = str(int(time.time()))
+        unique_title = f"Test Skill {ts}"
+
         # Upload the skill with the zip file
-        with create_skill_zip(skill_name) as zip_file:
+        with create_skill_zip(skill_name, unique_suffix=ts) as zip_file:
             response = litellm.create_skill(
                 display_title=unique_title,
                 files=[zip_file],
@@ -217,12 +234,13 @@ class BaseSkillsAPITest(ABC):
 
         # Use helper to create skill zip
         skill_name = "test-delete-skill"
-        
-        # Use unique title to avoid conflicts
-        unique_title = f"Test Delete Skill {int(time.time())}"
-        
+
+        # Use unique title and skill name to avoid conflicts
+        ts = str(int(time.time()))
+        unique_title = f"Test Delete Skill {ts}"
+
         # Create a skill specifically to delete
-        with create_skill_zip(skill_name) as zip_file:
+        with create_skill_zip(skill_name, unique_suffix=ts) as zip_file:
             created_skill = litellm.create_skill(
                 display_title=unique_title,
                 files=[zip_file],

tests/local_testing/test_amazing_vertex_completion.py

@@ -891,7 +891,7 @@ async def test_partner_models_httpx(model, region, sync_mode):
     "model,region",
     [
         # vertex_ai/meta/llama-4-scout-17b-16e-instruct-maas removed - consistently returns 400 BadRequest on Vertex AI
-        ("vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas", "us-south1"),
+        # vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas removed - us-south1 endpoint unavailable in CI
         (
             "vertex_ai/mistral-small-2503",
             "us-central1",

tests/local_testing/test_custom_callback_input.py

@@ -1085,7 +1085,10 @@ def test_standard_logging_payload(model, turn_off_message_logging):
         if turn_off_message_logging:
             print("checks redacted-by-litellm")
             assert "redacted-by-litellm" == slobject["messages"][0]["content"]
-            assert {"text": "redacted-by-litellm"} == slobject["response"]
+            # response is a full ModelResponse dict (choices format) since d84e5e381acf
+            response = slobject["response"]
+            assert response["choices"][0]["message"]["content"] == "redacted-by-litellm"
+            assert response["choices"][0]["message"].get("audio") is None
 
 
 @pytest.mark.parametrize(
@@ -1185,7 +1188,10 @@ def test_standard_logging_payload_audio(turn_off_message_logging, stream):
         if turn_off_message_logging:
             print("checks redacted-by-litellm")
             assert "redacted-by-litellm" == slobject["messages"][0]["content"]
-            assert {"text": "redacted-by-litellm"} == slobject["response"]
+            # response is a full ModelResponse dict (choices format) since d84e5e381acf
+            response = slobject["response"]
+            assert response["choices"][0]["message"]["content"] == "redacted-by-litellm"
+            assert response["choices"][0]["message"].get("audio") is None
 
 
 @pytest.mark.skip(reason="Works locally. Flaky on ci/cd")

tests/local_testing/test_stream_chunk_builder.py

@@ -636,6 +636,7 @@ def test_stream_chunk_builder_openai_prompt_caching():
             assert response_usage_value == v
 
 
+@pytest.mark.flaky(retries=5, delay=2)
 def test_stream_chunk_builder_openai_audio_output_usage():
     from pydantic import BaseModel
     from openai import OpenAI
@@ -666,13 +667,15 @@ def test_stream_chunk_builder_openai_audio_output_usage():
     usage_obj: Optional[litellm.Usage] = None
 
     for index, chunk in enumerate(chunks):
-        if hasattr(chunk, "usage"):
+        if hasattr(chunk, "usage") and chunk.usage is not None:
             usage_obj = chunk.usage
             print(f"chunk usage: {chunk.usage}")
             print(f"index: {index}")
             print(f"len chunks: {len(chunks)}")
 
     print(f"usage_obj: {usage_obj}")
+    if usage_obj is None:
+        pytest.skip("OpenAI did not return usage data in streaming response")
     response = stream_chunk_builder(chunks=chunks)
     print(f"response usage: {response.usage}")
     check_non_streaming_response(response)

tests/local_testing/test_streaming.py

@@ -3075,22 +3075,18 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk(
     """
     litellm.set_verbose = False
     chunks = [
-        litellm.ModelResponse(
-            **{
-                "id": "chatcmpl-123",
-                "object": "chat.completion.chunk",
-                "created": 1694268190,
-                "model": "gpt-3.5-turbo-0125",
-                "system_fingerprint": "fp_44709d6fcb",
-                "choices": [
-                    {
-                        "index": 0,
-                        "delta": {"content": chunk_value},
-                        "finish_reason": "stop",
-                    }
-                ],
-            },
-            stream=True,
+        litellm.ModelResponseStream(
+            id="chatcmpl-123",
+            created=1694268190,
+            model="gpt-3.5-turbo-0125",
+            system_fingerprint="fp_44709d6fcb",
+            choices=[
+                {
+                    "index": 0,
+                    "delta": {"content": chunk_value},
+                    "finish_reason": "stop",
+                }
+            ],
         )
     ] * loop_amount
     completion_stream = ModelResponseListIterator(model_responses=chunks)
@@ -3113,7 +3109,7 @@ def test_unit_test_custom_stream_wrapper_repeating_chunk(
     print(f"expected_chunk_fail: {expected_chunk_fail}")
 
     if (loop_amount > litellm.REPEATED_STREAMING_CHUNK_LIMIT) and expected_chunk_fail:
-        with pytest.raises(litellm.InternalServerError):
+        with pytest.raises((litellm.InternalServerError, litellm.exceptions.MidStreamFallbackError)):
             for chunk in response:
                 continue
     else:

tests/pass_through_tests/test_anthropic_passthrough.py

@@ -337,41 +337,46 @@ async def test_anthropic_messages_streaming_cost_injection():
     
     async with aiohttp.ClientSession() as session:
         async with session.post(
-            "http://0.0.0.0:4000/v1/messages", 
-            json=payload, 
-            headers=headers
+            "http://0.0.0.0:4000/v1/messages",
+            json=payload,
+            headers=headers,
         ) as response:
             assert response.status == 200
-            
-            # Collect all SSE events
+
+            # Collect all SSE events.
+            # Split each chunk by newlines to handle both:
+            # - Anthropic direct path: chunks arrive as individual lines
+            # - OpenAI/Responses API path: chunks are full multi-line SSE events
             events = []
-            async for line in response.content:
-                line_str = line.decode('utf-8').strip()
-                if line_str.startswith('data: '):
-                    try:
-                        data = json.loads(line_str[6:])  # Remove 'data: ' prefix
-                        events.append(data)
-                    except json.JSONDecodeError:
-                        continue
-            
+            async for chunk in response.content:
+                chunk_str = chunk.decode("utf-8")
+                for line in chunk_str.split("\n"):
+                    line = line.strip()
+                    if line.startswith("data: "):
+                        try:
+                            data = json.loads(line[6:])  # Remove 'data: ' prefix
+                            events.append(data)
+                        except json.JSONDecodeError:
+                            continue
+
             # Find message_delta event with usage
             message_delta_events = [
-                event for event in events 
-                if event.get('type') == 'message_delta' and 'usage' in event
+                event for event in events
+                if event.get("type") == "message_delta" and "usage" in event
             ]
-            
+
             assert len(message_delta_events) > 0, "No message_delta events with usage found"
-            
+
             # Check that cost is included in usage
             for event in message_delta_events:
-                usage = event.get('usage', {})
-                assert 'cost' in usage, f"Cost not found in usage: {usage}"
-                assert isinstance(usage['cost'], (int, float)), f"Cost should be numeric: {usage['cost']}"
-                assert usage['cost'] >= 0, f"Cost should be non-negative: {usage['cost']}"
-                
-                print(f"✅ Found message_delta with cost: {usage}")
-            
-            print(f"✅ Test passed: Found {len(message_delta_events)} message_delta events with cost")
+                usage = event.get("usage", {})
+                assert "cost" in usage, f"Cost not found in usage: {usage}"
+                assert isinstance(usage["cost"], (int, float)), f"Cost should be numeric: {usage['cost']}"
+                assert usage["cost"] >= 0, f"Cost should be non-negative: {usage['cost']}"
+
+                print(f"Found message_delta with cost: {usage}")
+
+            print(f"Test passed: Found {len(message_delta_events)} message_delta events with cost")
 
 
 @pytest.mark.asyncio
@@ -381,54 +386,61 @@ async def test_anthropic_messages_openai_model_streaming_cost_injection():
     Test that cost is injected into message_delta usage for OpenAI model via Anthropic Messages API
     """
     print("Testing cost injection in Anthropic Messages API with OpenAI model")
-    
+
     headers = {
         "Authorization": "Bearer sk-1234",
         "Content-Type": "application/json",
         "anthropic-version": "2023-06-01",
     }
-    
+
     payload = {
         "model": "openai/gpt-4o",
-        "max_tokens": 10,
+        "max_tokens": 20,
         "stream": True,
         "messages": [{"role": "user", "content": "Say 'Hi'"}],
     }
-    
+
     async with aiohttp.ClientSession() as session:
         async with session.post(
-            "http://0.0.0.0:4000/v1/messages", 
-            json=payload, 
-            headers=headers
+            "http://0.0.0.0:4000/v1/messages",
+            json=payload,
+            headers=headers,
         ) as response:
             assert response.status == 200
-            
-            # Collect all SSE events
+
+            # Collect all SSE events.
+            # Split each chunk by newlines to handle both:
+            # - Direct API paths: chunks arrive as individual lines
+            # - OpenAI/Responses API path: AnthropicResponsesStreamWrapper yields
+            #   full multi-line SSE events as single bytes objects, so a naive
+            #   startswith('data: ') check on the whole chunk misses them.
             events = []
-            async for line in response.content:
-                line_str = line.decode('utf-8').strip()
-                if line_str.startswith('data: '):
-                    try:
-                        data = json.loads(line_str[6:])  # Remove 'data: ' prefix
-                        events.append(data)
-                    except json.JSONDecodeError:
-                        continue
-            
+            async for chunk in response.content:
+                chunk_str = chunk.decode("utf-8")
+                for line in chunk_str.split("\n"):
+                    line = line.strip()
+                    if line.startswith("data: "):
+                        try:
+                            data = json.loads(line[6:])  # Remove 'data: ' prefix
+                            events.append(data)
+                        except json.JSONDecodeError:
+                            continue
+
             # Find message_delta event with usage
             message_delta_events = [
-                event for event in events 
-                if event.get('type') == 'message_delta' and 'usage' in event
+                event for event in events
+                if event.get("type") == "message_delta" and "usage" in event
             ]
-            
+
             assert len(message_delta_events) > 0, "No message_delta events with usage found"
-            
+
             # Check that cost is included in usage
             for event in message_delta_events:
-                usage = event.get('usage', {})
-                assert 'cost' in usage, f"Cost not found in usage: {usage}"
-                assert isinstance(usage['cost'], (int, float)), f"Cost should be numeric: {usage['cost']}"
-                assert usage['cost'] >= 0, f"Cost should be non-negative: {usage['cost']}"
-                
-                print(f"✅ Found message_delta with cost: {usage}")
-            
-            print(f"✅ Test passed: Found {len(message_delta_events)} message_delta events with cost")
+                usage = event.get("usage", {})
+                assert "cost" in usage, f"Cost not found in usage: {usage}"
+                assert isinstance(usage["cost"], (int, float)), f"Cost should be numeric: {usage['cost']}"
+                assert usage["cost"] >= 0, f"Cost should be non-negative: {usage['cost']}"
+
+                print(f"Found message_delta with cost: {usage}")
+
+            print(f"Test passed: Found {len(message_delta_events)} message_delta events with cost")

tests/proxy_admin_ui_tests/package.json

@@ -13,8 +13,8 @@
   },
   "overrides": {
     "glob": ">=11.1.0",
-    "tar": ">=7.5.8",
-    "minimatch": ">=10.2.1",
+    "tar": ">=7.5.10",
+    "minimatch": ">=10.2.4",
     "diff": ">=8.0.3",
     "@isaacs/brace-expansion": ">=5.0.1",
     "@babel/traverse": ">=7.23.2",

tests/proxy_admin_ui_tests/ui_unit_tests/package.json

@@ -25,8 +25,8 @@
     },
     "overrides": {
         "glob": ">=11.1.0",
-        "tar": ">=7.5.8",
-        "minimatch": ">=10.2.1",
+        "tar": ">=7.5.10",
+        "minimatch": ">=10.2.4",
         "diff": ">=8.0.3",
         "@isaacs/brace-expansion": ">=5.0.1",
         "@babel/traverse": ">=7.23.2",

tests/proxy_e2e_anthropic_messages_tests/test_claude_agent_sdk.py

@@ -4,7 +4,7 @@ E2E tests for Claude Agent SDK with LiteLLM Proxy using Bedrock models.
 Tests streaming messages across different Bedrock models:
 - Regular Bedrock Claude Sonnet 4.5
 - Bedrock Converse Claude Sonnet 4.5
-- AWS Nova Premier
+- AWS Nova Pro
 """
 
 import os
@@ -14,14 +14,14 @@ from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
 
 
 # Test models from test_config.yaml
-# Note: bedrock-converse-claude-sonnet-4.5 removed temporarily as the Bedrock Converse API 
+# Note: bedrock-converse-claude-sonnet-4.5 removed temporarily as the Bedrock Converse API
 # for Claude Sonnet 4.5 may not be available in all regions/accounts
-# Note: bedrock-nova-premier requires an inference profile for on-demand throughput
-# https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles.html
+# Note: bedrock-nova-premier requires provisioned throughput (not standard cross-region
+# inference profile) and is not reliably available in CI accounts. Using nova-pro instead.
 TEST_MODELS = [
     ("bedrock-claude-sonnet-4.5", "Bedrock Invoke API"),
     ("bedrock-converse-claude-sonnet-4.5", "Bedrock Converse API"),
-    ("bedrock-nova-premier", "AWS Nova Premier"),
+    ("bedrock-nova-pro", "AWS Nova Pro"),
 ]
 
 

tests/proxy_e2e_anthropic_messages_tests/test_config.yaml

@@ -24,9 +24,9 @@ model_list:
       model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
       aws_region_name: "us-east-1"
 
-  - model_name: bedrock-nova-premier
+  - model_name: bedrock-nova-pro
     litellm_params:
-      model: "bedrock/us.amazon.nova-premier-v1:0"
+      model: "bedrock/us.amazon.nova-pro-v1:0"
       aws_region_name: "us-east-1"
 
   # Converse API models
@@ -53,4 +53,5 @@ general_settings:
   forward_client_headers_to_llm_api: true
 
 litellm_settings:
-  drop_params: true
+  drop_params: true
+  modify_params: true

tests/search_tests/test_searxng_search.py

@@ -1,110 +1,327 @@
-import pytest
-import litellm
+"""
+Unit tests for SearXNG Search request/response transformation.
+
+These tests validate the request payload and response parsing without
+requiring a live SearXNG instance.
+"""
+
+import json
 import os
-from typing import List, Union
+from unittest.mock import MagicMock, patch
+from urllib.parse import parse_qs, urlparse
 
-from tests.search_tests.base_search_unit_tests import BaseSearchTest
+import httpx
+import pytest
+
+from litellm.llms.searxng.search.transformation import SearXNGSearchConfig
 
 
-class TestSearXNGSearch(BaseSearchTest):
+class TestSearXNGSearchRequestTransformation:
     """
-    Tests for SearXNG Search functionality.
+    Tests that SearXNG search requests are transformed into the expected payload.
     """
-    
-    def get_search_provider(self) -> str:
-        """
-        Return search_provider for SearXNG Search.
-        """
-        return "searxng"
-    
-    @pytest.mark.asyncio
-    async def test_basic_search(self):
-        """
-        Test basic search functionality with a simple query.
-        Override to handle free (0.0 cost) provider.
-        """
-        os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
-        litellm.model_cost = litellm.get_model_cost_map(url="")
-        litellm._turn_on_debug()
-        search_provider = self.get_search_provider()
-        print("Search Provider=", search_provider)
 
-        try:
-            response = await litellm.asearch(
-                query="latest developments in AI",
-                search_provider=search_provider,
-            )
-            print("Search response=", response.model_dump_json(indent=4))
+    def setup_method(self):
+        self.config = SearXNGSearchConfig()
 
-            print(f"\n{'='*80}")
-            print(f"Response type: {type(response)}")
-            print(f"Response object: {response.object if hasattr(response, 'object') else 'N/A'}")
-            
-            # Check if response has expected Search format
-            assert hasattr(response, "results"), "Response should have 'results' attribute"
-            assert hasattr(response, "object"), "Response should have 'object' attribute"
-            assert response.object == "search", f"Expected object='search', got '{response.object}'"
-            
-            # Validate results structure
-            assert isinstance(response.results, list), "results should be a list"
-            assert len(response.results) > 0, "Should have at least one result"
-            
-            # Check first result structure
-            first_result = response.results[0]
-            assert hasattr(first_result, "title"), "Result should have 'title' attribute"
-            assert hasattr(first_result, "url"), "Result should have 'url' attribute"
-            assert hasattr(first_result, "snippet"), "Result should have 'snippet' attribute"
-            
-            print(f"Total results: {len(response.results)}")
-            print(f"First result title: {first_result.title}")
-            print(f"First result URL: {first_result.url}")
-            print(f"First result snippet: {first_result.snippet[:100]}...")
-            print(f"{'='*80}\n")
-            
-            assert len(first_result.title) > 0, "Title should not be empty"
-            assert len(first_result.url) > 0, "URL should not be empty"
-            assert len(first_result.snippet) > 0, "Snippet should not be empty"
-            
-            # Validate cost tracking in _hidden_params
-            # For SearXNG (free provider), cost can be None or 0.0
-            assert hasattr(response, "_hidden_params"), "Response should have '_hidden_params' attribute"
-            hidden_params = response._hidden_params
-            assert "response_cost" in hidden_params, "_hidden_params should contain 'response_cost'"
-            
-            response_cost = hidden_params["response_cost"]
-            # SearXNG is free, so cost can be None or 0.0
-            if response_cost is not None:
-                assert isinstance(response_cost, (int, float)), "response_cost should be a number"
-                assert response_cost >= 0, "response_cost should be non-negative"
-                print(f"Cost tracking: ${response_cost:.6f}")
-            else:
-                print(f"Cost tracking: Free (None)")
-            
-        except Exception as e:
-            pytest.fail(f"Search call failed: {str(e)}")
-    
-    @pytest.mark.flaky(retries=3, delay=5)
-    def test_search_with_optional_params(self):
-        """
-        Test search with optional parameters.
-        Override for SearXNG since it doesn't natively limit results.
-        """
-        litellm.set_verbose = True
-        search_provider = self.get_search_provider()
-
-        response = litellm.search(
-            query="machine learning",
-            search_provider=search_provider,
-            max_results=5,
+    def test_basic_query_request(self):
+        """Test that a basic query produces the expected SearXNG request params."""
+        result = self.config.transform_search_request(
+            query="artificial intelligence recent news",
+            optional_params={},
         )
 
-        # Validate response
-        assert hasattr(response, "results"), "Response should have 'results' attribute"
-        assert isinstance(response.results, list), "results should be a list"
-        assert len(response.results) > 0, "Should have at least one result"
-        # Note: SearXNG doesn't natively limit results, so we don't check <= 5
-        
-        print(f"\nSearch with optional params validated:")
-        print(f"  - Requested max_results: 5")
-        print(f"  - Received results: {len(response.results)}")
+        assert "_searxng_params" in result
+        params = result["_searxng_params"]
+        assert params["q"] == "artificial intelligence recent news"
+        assert params["format"] == "json"
 
+    def test_list_query_joined(self):
+        """Test that a list query is joined into a single string."""
+        result = self.config.transform_search_request(
+            query=["artificial intelligence", "recent news"],
+            optional_params={},
+        )
+
+        params = result["_searxng_params"]
+        assert params["q"] == "artificial intelligence recent news"
+        assert params["format"] == "json"
+
+    def test_country_to_language_mapping(self):
+        """Test that country codes are mapped to SearXNG language params."""
+        test_cases = {
+            "us": "en",
+            "uk": "en",
+            "de": "de",
+            "fr": "fr",
+            "es": "es",
+            "jp": "ja",
+            "br": "br",  # unmapped country passed through as-is
+        }
+        for country, expected_language in test_cases.items():
+            result = self.config.transform_search_request(
+                query="test",
+                optional_params={"country": country},
+            )
+            params = result["_searxng_params"]
+            assert params["language"] == expected_language, (
+                f"country={country} should map to language={expected_language}"
+            )
+
+    def test_max_results_ignored(self):
+        """Test that max_results is accepted but doesn't add extra params."""
+        result = self.config.transform_search_request(
+            query="test",
+            optional_params={"max_results": 5},
+        )
+
+        params = result["_searxng_params"]
+        assert params["q"] == "test"
+        assert params["format"] == "json"
+        # max_results should not appear in the SearXNG params
+        assert "max_results" not in params
+
+    def test_searxng_specific_params_passthrough(self):
+        """Test that SearXNG-specific params are passed through as-is."""
+        result = self.config.transform_search_request(
+            query="test",
+            optional_params={"categories": "general,news", "engines": "google,bing", "time_range": "month"},
+        )
+
+        params = result["_searxng_params"]
+        assert params["q"] == "test"
+        assert params["format"] == "json"
+        assert params["categories"] == "general,news"
+        assert params["engines"] == "google,bing"
+        assert params["time_range"] == "month"
+
+
+class TestSearXNGSearchURLConstruction:
+    """
+    Tests that the complete URL is built correctly from api_base and request params.
+    """
+
+    def setup_method(self):
+        self.config = SearXNGSearchConfig()
+
+    def test_url_with_search_suffix(self):
+        """Test URL construction appends /search."""
+        data = {"_searxng_params": {"q": "test query", "format": "json"}}
+        url = self.config.get_complete_url(
+            api_base="https://searxng.example.com",
+            optional_params={},
+            data=data,
+        )
+
+        parsed = urlparse(url)
+        assert parsed.scheme == "https"
+        assert parsed.netloc == "searxng.example.com"
+        assert parsed.path == "/search"
+        query_params = parse_qs(parsed.query)
+        assert query_params["q"] == ["test query"]
+        assert query_params["format"] == ["json"]
+
+    def test_url_already_has_search_suffix(self):
+        """Test URL construction doesn't double-append /search."""
+        data = {"_searxng_params": {"q": "test", "format": "json"}}
+        url = self.config.get_complete_url(
+            api_base="https://searxng.example.com/search",
+            optional_params={},
+            data=data,
+        )
+
+        parsed = urlparse(url)
+        assert parsed.path == "/search"
+        assert "/search/search" not in url
+
+    def test_url_with_trailing_slash(self):
+        """Test URL construction with trailing slash on api_base."""
+        data = {"_searxng_params": {"q": "test", "format": "json"}}
+        url = self.config.get_complete_url(
+            api_base="https://searxng.example.com/",
+            optional_params={},
+            data=data,
+        )
+
+        parsed = urlparse(url)
+        assert parsed.path == "/search"
+
+    def test_url_from_env_variable(self):
+        """Test URL construction falls back to SEARXNG_API_BASE env var."""
+        data = {"_searxng_params": {"q": "test", "format": "json"}}
+        with patch(
+            "litellm.llms.searxng.search.transformation.get_secret_str",
+            return_value="https://env-searxng.example.com",
+        ):
+            url = self.config.get_complete_url(
+                api_base=None,
+                optional_params={},
+                data=data,
+            )
+
+        assert url.startswith("https://env-searxng.example.com/search?")
+
+    def test_url_missing_api_base_raises(self):
+        """Test that missing api_base and env var raises ValueError."""
+        with patch(
+            "litellm.llms.searxng.search.transformation.get_secret_str",
+            return_value=None,
+        ):
+            with pytest.raises(ValueError, match="SEARXNG_API_BASE is not set"):
+                self.config.get_complete_url(
+                    api_base=None,
+                    optional_params={},
+                    data={"_searxng_params": {"q": "test"}},
+                )
+
+    def test_url_without_data_returns_base(self):
+        """Test URL construction without data returns just the api_base/search."""
+        url = self.config.get_complete_url(
+            api_base="https://searxng.example.com",
+            optional_params={},
+            data=None,
+        )
+
+        assert url == "https://searxng.example.com/search"
+
+
+class TestSearXNGSearchResponseTransformation:
+    """
+    Tests that SearXNG API responses are correctly transformed to SearchResponse.
+    """
+
+    def setup_method(self):
+        self.config = SearXNGSearchConfig()
+        self.logging_obj = MagicMock()
+
+    def _make_mock_response(self, json_data: dict) -> httpx.Response:
+        response = httpx.Response(
+            status_code=200,
+            json=json_data,
+            request=httpx.Request("GET", "https://searxng.example.com/search"),
+        )
+        return response
+
+    def test_response_with_results(self):
+        """Test transforming a typical SearXNG response with results."""
+        raw = self._make_mock_response({
+            "results": [
+                {
+                    "title": "AI News Article",
+                    "url": "https://example.com/ai-news",
+                    "content": "Latest developments in artificial intelligence.",
+                    "publishedDate": "2025-01-15",
+                },
+                {
+                    "title": "ML Research Paper",
+                    "url": "https://example.com/ml-paper",
+                    "content": "New machine learning research findings.",
+                    "pubdate": "2025-01-10",
+                },
+            ]
+        })
+
+        response = self.config.transform_search_response(
+            raw_response=raw, logging_obj=self.logging_obj
+        )
+
+        assert response.object == "search"
+        assert len(response.results) == 2
+
+        first = response.results[0]
+        assert first.title == "AI News Article"
+        assert first.url == "https://example.com/ai-news"
+        assert first.snippet == "Latest developments in artificial intelligence."
+        assert first.date == "2025-01-15"
+        assert first.last_updated is None
+
+        second = response.results[1]
+        assert second.title == "ML Research Paper"
+        assert second.date == "2025-01-10"  # from pubdate field
+
+    def test_response_empty_results(self):
+        """Test transforming a response with no results."""
+        raw = self._make_mock_response({"results": []})
+
+        response = self.config.transform_search_response(
+            raw_response=raw, logging_obj=self.logging_obj
+        )
+
+        assert response.object == "search"
+        assert response.results == []
+
+    def test_response_missing_results_key(self):
+        """Test transforming a response that has no 'results' key."""
+        raw = self._make_mock_response({"query": "test"})
+
+        response = self.config.transform_search_response(
+            raw_response=raw, logging_obj=self.logging_obj
+        )
+
+        assert response.object == "search"
+        assert response.results == []
+
+    def test_response_missing_optional_fields(self):
+        """Test transforming results with missing optional fields."""
+        raw = self._make_mock_response({
+            "results": [
+                {
+                    "title": "Minimal Result",
+                    "url": "https://example.com",
+                }
+            ]
+        })
+
+        response = self.config.transform_search_response(
+            raw_response=raw, logging_obj=self.logging_obj
+        )
+
+        result = response.results[0]
+        assert result.title == "Minimal Result"
+        assert result.url == "https://example.com"
+        assert result.snippet == ""  # defaults to empty string
+        assert result.date is None
+        assert result.last_updated is None
+
+
+class TestSearXNGSearchHeaders:
+    """
+    Tests for header/environment validation.
+    """
+
+    def setup_method(self):
+        self.config = SearXNGSearchConfig()
+
+    def test_headers_without_api_key(self):
+        """Test that headers are set correctly without an API key."""
+        with patch(
+            "litellm.llms.searxng.search.transformation.get_secret_str",
+            return_value=None,
+        ):
+            headers = self.config.validate_environment(headers={})
+
+        assert headers["Content-Type"] == "application/json"
+        assert "Authorization" not in headers
+
+    def test_headers_with_api_key(self):
+        """Test that headers include Authorization when API key is provided."""
+        headers = self.config.validate_environment(
+            headers={}, api_key="test-key-123"
+        )
+
+        assert headers["Content-Type"] == "application/json"
+        assert headers["Authorization"] == "Bearer test-key-123"
+
+    def test_headers_with_env_api_key(self):
+        """Test that headers use SEARXNG_API_KEY from env."""
+        with patch(
+            "litellm.llms.searxng.search.transformation.get_secret_str",
+            return_value="env-key-456",
+        ):
+            headers = self.config.validate_environment(headers={})
+
+        assert headers["Authorization"] == "Bearer env-key-456"
+
+    def test_http_method_is_get(self):
+        """Test that the HTTP method is GET."""
+        assert self.config.get_http_method() == "GET"

tests/search_tests/test_serper_search.py

@@ -0,0 +1,184 @@
+"""
+Tests for Serper Search API integration.
+"""
+import os
+import sys
+import pytest
+from unittest.mock import AsyncMock, patch, MagicMock
+
+sys.path.insert(
+    0, os.path.abspath("../..")
+)
+
+import litellm
+
+
+class TestSerperSearch:
+    """
+    Tests for Serper Search functionality with mocked network responses.
+    """
+    
+    @pytest.mark.asyncio
+    async def test_serper_search_request_payload(self):
+        """
+        Test that validates the Serper search request payload structure without making real API calls.
+        """
+        # Set environment variable for API key
+        os.environ["SERPER_API_KEY"] = "test-api-key"
+        
+        # Create a mock response
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "organic": [
+                {
+                    "title": "Test Result 1",
+                    "link": "https://example.com/1",
+                    "snippet": "This is a test snippet for result 1",
+                    "position": 1,
+                },
+                {
+                    "title": "Test Result 2",
+                    "link": "https://example.com/2",
+                    "snippet": "This is a test snippet for result 2",
+                    "position": 2,
+                    "date": "Jan 15, 2025",
+                },
+            ],
+        }
+        
+        # Mock the httpx AsyncClient post method
+        with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock) as mock_post:
+            mock_post.return_value = mock_response
+            
+            # Make the search call
+            response = await litellm.asearch(
+                query="latest developments in AI",
+                search_provider="serper",
+                max_results=5
+            )
+            
+            # Verify the post method was called once
+            assert mock_post.call_count == 1
+            
+            # Get the actual call arguments
+            call_args = mock_post.call_args
+            
+            # Verify URL
+            assert call_args.kwargs["url"] == "https://google.serper.dev/search"
+            
+            # Verify headers contain X-API-KEY
+            headers = call_args.kwargs.get("headers", {})
+            assert "X-API-KEY" in headers
+            assert headers["X-API-KEY"] == "test-api-key"
+            assert headers["Content-Type"] == "application/json"
+            
+            # Verify request payload
+            json_data = call_args.kwargs.get("json")
+            assert json_data is not None
+            assert json_data["q"] == "latest developments in AI"
+            assert json_data["num"] == 5
+            
+            # Verify response structure
+            assert hasattr(response, "results")
+            assert hasattr(response, "object")
+            assert response.object == "search"
+            assert len(response.results) == 2
+            
+            # Verify first result
+            first_result = response.results[0]
+            assert first_result.title == "Test Result 1"
+            assert first_result.url == "https://example.com/1"
+            assert first_result.snippet == "This is a test snippet for result 1"
+            
+            # Verify date on second result
+            second_result = response.results[1]
+            assert second_result.date == "Jan 15, 2025"
+
+    @pytest.mark.asyncio
+    async def test_serper_search_with_country(self):
+        """
+        Test that country parameter is mapped to 'gl' in Serper request.
+        """
+        os.environ["SERPER_API_KEY"] = "test-api-key"
+        
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "organic": [
+                {
+                    "title": "Result",
+                    "link": "https://example.com",
+                    "snippet": "Snippet",
+                }
+            ]
+        }
+        
+        with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock) as mock_post:
+            mock_post.return_value = mock_response
+            
+            await litellm.asearch(
+                query="test query",
+                search_provider="serper",
+                country="US",
+            )
+            
+            json_data = mock_post.call_args.kwargs.get("json")
+            assert json_data["gl"] == "us"
+
+    @pytest.mark.asyncio
+    async def test_serper_search_with_domain_filter(self):
+        """
+        Test that search_domain_filter is appended as site: clauses to the query.
+        """
+        os.environ["SERPER_API_KEY"] = "test-api-key"
+        
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "organic": [
+                {
+                    "title": "Result",
+                    "link": "https://arxiv.org/paper/1",
+                    "snippet": "Snippet",
+                }
+            ]
+        }
+        
+        with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock) as mock_post:
+            mock_post.return_value = mock_response
+            
+            await litellm.asearch(
+                query="machine learning",
+                search_provider="serper",
+                search_domain_filter=["arxiv.org", "nature.com"],
+            )
+            
+            json_data = mock_post.call_args.kwargs.get("json")
+            assert "site:arxiv.org" in json_data["q"]
+            assert "site:nature.com" in json_data["q"]
+            assert "machine learning" in json_data["q"]
+
+    @pytest.mark.asyncio
+    async def test_serper_search_empty_organic(self):
+        """
+        Test handling of response with no organic results.
+        """
+        os.environ["SERPER_API_KEY"] = "test-api-key"
+        
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {
+            "searchParameters": {"q": "xyznonexistent"},
+        }
+        
+        with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=AsyncMock) as mock_post:
+            mock_post.return_value = mock_response
+            
+            response = await litellm.asearch(
+                query="xyznonexistent",
+                search_provider="serper",
+            )
+            
+            assert response.object == "search"
+            assert len(response.results) == 0

tests/test_litellm/llms/anthropic/test_v1_messages_empty_text_sanitization.py

@@ -0,0 +1,247 @@
+"""
+Test empty text content block sanitization for the /v1/messages native path.
+
+The Anthropic API returns assistant messages with empty text blocks
+({"type": "text", "text": ""}) alongside tool_use blocks, but rejects
+them when sent back. The /v1/messages endpoint must strip these before
+forwarding to providers.
+
+Ref: https://github.com/BerriAI/litellm/issues/22930
+"""
+
+import pytest
+
+from litellm.llms.custom_httpx.llm_http_handler import (
+    _sanitize_anthropic_messages_empty_text_blocks,
+)
+
+
+class TestSanitizeAnthropicMessagesEmptyTextBlocks:
+    """Unit tests for _sanitize_anthropic_messages_empty_text_blocks."""
+
+    def test_strips_empty_text_alongside_tool_use(self):
+        """
+        The most common case from the bug report: an assistant message
+        containing an empty text block next to a tool_use block.
+        """
+        messages = [
+            {"role": "user", "content": "Run the command."},
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": ""},
+                    {
+                        "type": "tool_use",
+                        "id": "toolu_xxx",
+                        "name": "Bash",
+                        "input": {"command": "ls"},
+                    },
+                ],
+            },
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert len(result) == 2
+        assert result[0] == messages[0]  # user message unchanged
+        # assistant content should only have the tool_use block
+        assert len(result[1]["content"]) == 1
+        assert result[1]["content"][0]["type"] == "tool_use"
+
+    def test_preserves_nonempty_text_blocks(self):
+        """Non-empty text blocks must not be removed."""
+        messages = [
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": "Let me check that."},
+                    {
+                        "type": "tool_use",
+                        "id": "toolu_yyy",
+                        "name": "Bash",
+                        "input": {"command": "pwd"},
+                    },
+                ],
+            },
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert len(result[0]["content"]) == 2
+        assert result[0]["content"][0] == {"type": "text", "text": "Let me check that."}
+
+    def test_whitespace_only_text_block_stripped(self):
+        """Whitespace-only text blocks should also be stripped."""
+        messages = [
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": "   \n\t  "},
+                    {
+                        "type": "tool_use",
+                        "id": "toolu_zzz",
+                        "name": "Bash",
+                        "input": {},
+                    },
+                ],
+            },
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert len(result[0]["content"]) == 1
+        assert result[0]["content"][0]["type"] == "tool_use"
+
+    def test_all_empty_text_blocks_replaced_with_placeholder(self):
+        """
+        If all content blocks are empty text, replace with a placeholder
+        to avoid sending an empty content array.
+        """
+        messages = [
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": ""},
+                ],
+            },
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert len(result[0]["content"]) == 1
+        assert result[0]["content"][0]["type"] == "text"
+        assert result[0]["content"][0]["text"].strip()  # must be non-empty
+
+    def test_string_content_untouched(self):
+        """Messages with string content should pass through unchanged."""
+        messages = [
+            {"role": "user", "content": "Hello"},
+            {"role": "assistant", "content": "Hi there!"},
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert result == messages
+
+    def test_no_content_key_untouched(self):
+        """Messages without a content key should pass through."""
+        messages = [
+            {"role": "user", "content": "Hello"},
+            {"role": "assistant"},
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert result == messages
+
+    def test_user_message_content_list_also_sanitized(self):
+        """
+        Empty text blocks should be stripped from user messages too,
+        not just assistant messages.
+        """
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": ""},
+                    {"type": "text", "text": "actual question"},
+                ],
+            },
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert len(result[0]["content"]) == 1
+        assert result[0]["content"][0]["text"] == "actual question"
+
+    def test_tool_result_content_blocks_untouched(self):
+        """
+        tool_result content blocks should not be affected — only
+        {"type": "text", "text": ""} blocks are stripped.
+        """
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "toolu_xxx",
+                        "content": "",
+                    },
+                ],
+            },
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        assert result == messages
+
+    def test_multiple_messages_mixed(self):
+        """End-to-end scenario with multiple messages, some needing sanitization."""
+        messages = [
+            {"role": "user", "content": "Run ls"},
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": ""},
+                    {
+                        "type": "tool_use",
+                        "id": "toolu_1",
+                        "name": "Bash",
+                        "input": {"command": "ls"},
+                    },
+                ],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "tool_result",
+                        "tool_use_id": "toolu_1",
+                        "content": "file1.txt\nfile2.txt",
+                    },
+                ],
+            },
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": "Here are the files:"},
+                ],
+            },
+        ]
+
+        result = _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        # First message: string content, unchanged
+        assert result[0] == messages[0]
+        # Second message: empty text stripped, only tool_use remains
+        assert len(result[1]["content"]) == 1
+        assert result[1]["content"][0]["type"] == "tool_use"
+        # Third message: tool_result, unchanged
+        assert result[2] == messages[2]
+        # Fourth message: non-empty text, unchanged
+        assert result[3] == messages[3]
+
+    def test_does_not_mutate_original_messages(self):
+        """The function should not modify the input list or its dicts."""
+        original_content = [
+            {"type": "text", "text": ""},
+            {
+                "type": "tool_use",
+                "id": "toolu_1",
+                "name": "Bash",
+                "input": {},
+            },
+        ]
+        messages = [
+            {
+                "role": "assistant",
+                "content": original_content,
+            },
+        ]
+
+        _sanitize_anthropic_messages_empty_text_blocks(messages)
+
+        # Original message content should be unchanged
+        assert len(messages[0]["content"]) == 2
+        assert messages[0]["content"][0] == {"type": "text", "text": ""}

tests/test_litellm/llms/bedrock/chat/agentcore/test_agentcore_transformation.py

@@ -1,20 +1,22 @@
 """
-Unit tests for Bedrock AgentCore transformation — Accept header fix.
+Unit tests for Bedrock AgentCore transformation.
 
-Verifies that AmazonAgentCoreConfig.sign_request() sets the
-Accept: application/json, text/event-stream header required by
-MCP servers on Bedrock AgentCore.
+Tests:
+- Accept header fix (sign_request sets Accept: application/json, text/event-stream)
+- JSON response parsing fallback chain (_parse_json_response supports multiple schemas)
+- Streaming Content-Type fallback (JSON responses converted to single-chunk streams)
 """
 
 import json
 import os
 import sys
 
+import httpx
 import pytest
 
 sys.path.insert(0, os.path.abspath("../../../../../.."))
 
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, Mock, patch
 
 import litellm
 from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
@@ -81,3 +83,237 @@ class TestAgentCoreAcceptHeader:
             headers = mock_post.call_args.kwargs["headers"]
             assert "Accept" in headers
             assert headers["Accept"] == "application/json, text/event-stream"
+
+
+class TestAgentCoreJsonResponseParsing:
+    """Tests for _parse_json_response fallback chain."""
+
+    @pytest.fixture
+    def config(self):
+        return AmazonAgentCoreConfig()
+
+    def test_parse_json_standard_agentcore_format(self, config):
+        """Strategy 1: standard {"result": {"content": [{"text": "..."}]}} format."""
+        response_json = {
+            "result": {
+                "role": "assistant",
+                "content": [{"text": "Hello from standard format"}],
+            }
+        }
+        parsed = config._parse_json_response(response_json)
+        assert parsed["content"] == "Hello from standard format"
+        assert parsed["usage"] is None
+        assert parsed["final_message"] == response_json["result"]
+
+    def test_parse_json_strands_format(self, config):
+        """Strategy 2: Strands {"response": [{"text": "..."}]} format."""
+        response_json = {
+            "response": [
+                {"text": "Based on my research, "},
+                {"text": "iOS 18.2 was released."},
+            ]
+        }
+        parsed = config._parse_json_response(response_json)
+        assert parsed["content"] == "Based on my research, iOS 18.2 was released."
+        assert parsed["usage"] is None
+        assert parsed["final_message"] is None
+
+    def test_parse_json_string_result(self, config):
+        """Strategy 3: plain string {"result": "text"} format."""
+        response_json = {"result": "Simple text response"}
+        parsed = config._parse_json_response(response_json)
+        assert parsed["content"] == "Simple text response"
+        assert parsed["usage"] is None
+
+    def test_parse_json_string_response(self, config):
+        """Strategy 3: plain string {"response": "text"} format."""
+        response_json = {"response": "Another text response"}
+        parsed = config._parse_json_response(response_json)
+        assert parsed["content"] == "Another text response"
+        assert parsed["usage"] is None
+
+    def test_parse_json_unknown_format_fallback(self, config):
+        """Strategy 4: unknown keys fall back to raw JSON."""
+        response_json = {"custom_key": "custom_value", "data": [1, 2, 3]}
+        parsed = config._parse_json_response(response_json)
+        assert parsed["content"] == json.dumps(response_json)
+        assert parsed["usage"] is None
+        assert parsed["final_message"] is None
+
+    def test_parse_json_non_dict_response(self, config):
+        """Guard: non-dict JSON (e.g. array) falls back to raw JSON string."""
+        response_json = [{"text": "array response"}]
+        parsed = config._parse_json_response(response_json)
+        assert parsed["content"] == json.dumps(response_json)
+        assert parsed["usage"] is None
+        assert parsed["final_message"] is None
+
+    def test_parse_json_empty_content_in_result(self, config):
+        """Standard format with empty content list - preserves existing behavior."""
+        response_json = {
+            "result": {
+                "role": "assistant",
+                "content": [],
+            }
+        }
+        parsed = config._parse_json_response(response_json)
+        assert parsed["content"] == ""
+        assert parsed["final_message"] == response_json["result"]
+
+
+class TestAgentCoreNonStreamingJsonFormats:
+    """Tests for _get_parsed_response with different JSON formats (non-streaming path)."""
+
+    @pytest.fixture
+    def config(self):
+        return AmazonAgentCoreConfig()
+
+    def test_get_parsed_response_strands_json(self, config):
+        """
+        Non-streaming path: _get_parsed_response routes application/json
+        to _parse_json_response which handles the Strands format.
+        """
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.headers = {"content-type": "application/json"}
+        mock_response.json.return_value = {
+            "response": [{"text": "Strands agent response via non-streaming"}]
+        }
+        parsed = config._get_parsed_response(mock_response)
+        assert parsed["content"] == "Strands agent response via non-streaming"
+        assert parsed["usage"] is None
+
+    def test_get_parsed_response_raw_json_fallback(self, config):
+        """
+        Non-streaming path: unknown JSON schema falls back to raw JSON string.
+        """
+        response_json = {"output": "some value"}
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.headers = {"content-type": "application/json"}
+        mock_response.json.return_value = response_json
+        parsed = config._get_parsed_response(mock_response)
+        assert parsed["content"] == json.dumps(response_json)
+
+
+class TestAgentCoreStreamingJsonFallback:
+    """Tests for streaming Content-Type check (JSON -> single-chunk stream)."""
+
+    def test_sync_streaming_with_json_response(self):
+        """
+        When stream=True but the agent returns Content-Type: application/json,
+        content is extracted and returned instead of silently returning empty.
+        Exercises the full path through litellm.completion().
+        """
+        from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+        client = HTTPHandler()
+        json_body = {"response": [{"text": "Strands sync response"}]}
+
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.status_code = 200
+        mock_response.headers = {"content-type": "application/json"}
+        mock_response.read.return_value = json.dumps(json_body).encode()
+
+        with patch.object(client, "post", return_value=mock_response):
+            response = litellm.completion(
+                model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent",
+                messages=[{"role": "user", "content": "test"}],
+                stream=True,
+                client=client,
+            )
+
+            # Collect content across all chunks
+            # CustomStreamWrapper yields content chunk(s) + a synthetic stop chunk
+            content = ""
+            for chunk in response:
+                if chunk.choices[0].delta.content:
+                    content += chunk.choices[0].delta.content
+
+        assert content == "Strands sync response"
+
+    async def test_async_streaming_with_json_response(self):
+        """
+        Async streaming: same Content-Type: application/json fallback via
+        litellm.acompletion(stream=True).
+        """
+        from unittest.mock import AsyncMock
+
+        from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+        client = AsyncHTTPHandler()
+        json_body = {"response": [{"text": "Strands async response"}]}
+
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.status_code = 200
+        mock_response.headers = {"content-type": "application/json"}
+        mock_response.aread = AsyncMock(
+            return_value=json.dumps(json_body).encode()
+        )
+
+        with patch.object(
+            client, "post", new_callable=AsyncMock, return_value=mock_response
+        ):
+            response = await litellm.acompletion(
+                model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent",
+                messages=[{"role": "user", "content": "test"}],
+                stream=True,
+                client=client,
+            )
+
+            # Collect content across all chunks
+            content = ""
+            async for chunk in response:
+                if chunk.choices[0].delta.content:
+                    content += chunk.choices[0].delta.content
+
+        assert content == "Strands async response"
+
+    def test_sync_streaming_malformed_json_raises_error(self):
+        """
+        When stream=True and Content-Type is application/json but the body
+        is malformed JSON, an error is raised with a descriptive message
+        (not a raw JSONDecodeError).
+        """
+        from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+        client = HTTPHandler()
+
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.status_code = 200
+        mock_response.headers = {"content-type": "application/json"}
+        mock_response.read.return_value = b"not valid json {{"
+
+        with patch.object(client, "post", return_value=mock_response):
+            with pytest.raises(Exception, match="Failed to read/parse JSON response body"):
+                litellm.completion(
+                    model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent",
+                    messages=[{"role": "user", "content": "test"}],
+                    stream=True,
+                    client=client,
+                )
+
+    async def test_async_streaming_malformed_json_raises_error(self):
+        """
+        Async mirror: malformed JSON body raises a structured error, not a
+        raw JSONDecodeError.
+        """
+        from unittest.mock import AsyncMock
+
+        from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+        client = AsyncHTTPHandler()
+
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.status_code = 200
+        mock_response.headers = {"content-type": "application/json"}
+        mock_response.aread = AsyncMock(return_value=b"not valid json {{")
+
+        with patch.object(
+            client, "post", new_callable=AsyncMock, return_value=mock_response
+        ):
+            with pytest.raises(Exception, match="Failed to read/parse JSON response body"):
+                await litellm.acompletion(
+                    model="bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/test_agent",
+                    messages=[{"role": "user", "content": "test"}],
+                    stream=True,
+                    client=client,
+                )

tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py

@@ -386,6 +386,40 @@ def test_opus_4_5_model_detection():
 #         f"computer-use beta should be kept, got: {anthropic_beta}"
 
 
+def test_output_config_removed_from_bedrock_chat_invoke_request():
+    """
+    Test that output_config parameter is stripped from Bedrock Chat Invoke requests.
+
+    Bedrock Invoke API doesn't support the output_config parameter (Anthropic-only).
+    Ensures the chat/invoke path mirrors the messages/invoke path fix.
+
+    Fixes: https://github.com/BerriAI/litellm/issues/22797
+    """
+    config = AmazonAnthropicClaudeConfig()
+
+    messages = [{"role": "user", "content": "test"}]
+
+    # Inject output_config into optional_params (simulates Anthropic SDK forwarding it)
+    optional_params = {
+        "max_tokens": 100,
+        "output_config": {"effort": "high"},
+    }
+
+    result = config.transform_request(
+        model="anthropic.claude-sonnet-4-20250514-v1:0",
+        messages=messages,
+        optional_params=optional_params,
+        litellm_params={},
+        headers={},
+    )
+
+    assert "output_config" not in result, (
+        f"output_config should be stripped for Bedrock Chat Invoke, got keys: {list(result.keys())}"
+    )
+    # Verify normal params survive
+    assert result["max_tokens"] == 100
+
+
 def test_output_format_removed_from_bedrock_invoke_request():
     """
     Test that output_format parameter is removed from Bedrock Invoke requests.

tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py

@@ -275,3 +275,70 @@ def test_remove_scope_from_cache_control():
     # Verify scope is removed from messages
     assert "scope" not in request["messages"][0]["content"][0]["cache_control"]
     assert request["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral"
+
+
+def test_bedrock_messages_strips_output_config():
+    """
+    Ensure output_config is stripped from the request before sending to
+    Bedrock Invoke, which doesn't support this Anthropic-specific parameter.
+
+    Regression test for: https://github.com/BerriAI/litellm/issues/22797
+    """
+    from litellm.types.router import GenericLiteLLMParams
+
+    cfg = AmazonAnthropicClaudeMessagesConfig()
+    messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+    optional_params = {
+        "max_tokens": 4096,
+        "output_config": {
+            "effort": "high",
+        },
+    }
+
+    result = cfg.transform_anthropic_messages_request(
+        model="anthropic.claude-3-haiku-20240307-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+
+    assert "output_config" not in result, (
+        "output_config should be stripped — Bedrock Invoke rejects it"
+    )
+    # Other params should be preserved
+    assert result.get("max_tokens") == 4096
+
+
+def test_bedrock_messages_strips_output_config_with_output_format():
+    """
+    When both output_config and output_format are present, both should be
+    stripped (output_format is converted to inline schema, output_config
+    is simply dropped).
+    """
+    from litellm.types.router import GenericLiteLLMParams
+
+    cfg = AmazonAnthropicClaudeMessagesConfig()
+    messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+    optional_params = {
+        "max_tokens": 4096,
+        "output_config": {"effort": "low"},
+        "output_format": {
+            "type": "json_schema",
+            "schema": {
+                "type": "object",
+                "properties": {"answer": {"type": "string"}},
+            },
+        },
+    }
+
+    result = cfg.transform_anthropic_messages_request(
+        model="anthropic.claude-3-haiku-20240307-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+
+    assert "output_config" not in result
+    assert "output_format" not in result

tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py

@@ -212,7 +212,7 @@ def test_build_vertex_schema():
         "properties": {
             "state": {
                 "properties": {
-                    "messages": {"items": {}, "type": "array"},
+                    "messages": {"items": {"type": "object"}, "type": "array"},
                     "conversation_id": {"type": "string"},
                 },
                 "required": ["messages", "conversation_id"],
@@ -226,7 +226,7 @@ def test_build_vertex_schema():
                     "callbacks": {
                         "anyOf": [
                             {"type": "array", "nullable": True},
-                            {"nullable": True},
+                            {"type": "object", "nullable": True},
                         ]
                     },
                     "run_name": {"type": "string"},
@@ -270,28 +270,23 @@ def test_process_items_basic():
     """Test basic functionality of process_items."""
     from litellm.llms.vertex_ai.common_utils import process_items
 
-    # Test empty items — should preserve "any type" semantics (not coerce to object)
+    # Test empty items
     schema = {"type": "array", "items": {}}
     process_items(schema)
-    assert schema["items"] == {}
+    assert schema["items"] == {"type": "object"}
 
-    # Test nested items — should preserve "any type" semantics
+    # Test nested items
     schema = {"type": "array", "items": {"type": "array", "items": {}}}
     process_items(schema)
-    assert schema["items"]["items"] == {}
+    assert schema["items"]["items"] == {"type": "object"}
 
-    # Test items in properties — should preserve "any type" semantics
+    # Test items in properties
     schema = {
         "type": "object",
         "properties": {"nested": {"type": "array", "items": {}}},
     }
     process_items(schema)
-    assert schema["properties"]["nested"]["items"] == {}
-
-    # Test items with actual type — should not be modified
-    schema = {"type": "array", "items": {"type": "string"}}
-    process_items(schema)
-    assert schema["items"] == {"type": "string"}
+    assert schema["properties"]["nested"]["items"] == {"type": "object"}
 
 
 def test_vertex_ai_complex_response_schema():
@@ -1407,89 +1402,3 @@ def test_add_object_type_does_not_add_type_when_anyof_present():
 
     # Verify type was not added (anyOf handles the type)
     assert "type" not in input_schema, "type should not be added when anyOf is present"
-
-
-def test_is_any_type_schema():
-    """Test _is_any_type_schema correctly identifies unconstrained schemas."""
-    from litellm.llms.vertex_ai.common_utils import _is_any_type_schema
-
-    # Empty schema = any type
-    assert _is_any_type_schema({}) is True
-
-    # Only metadata keys = any type
-    assert _is_any_type_schema({"description": "Any value"}) is True
-    assert _is_any_type_schema({"title": "MyField"}) is True
-    assert _is_any_type_schema({"title": "X", "description": "Y", "default": 0}) is True
-
-    # Has type-constraining keys = NOT any type
-    assert _is_any_type_schema({"type": "object"}) is False
-    assert _is_any_type_schema({"type": "string"}) is False
-    assert _is_any_type_schema({"properties": {"a": {}}}) is False
-    assert _is_any_type_schema({"items": {"type": "string"}}) is False
-    assert _is_any_type_schema({"anyOf": [{"type": "string"}]}) is False
-    assert _is_any_type_schema({"$schema": "https://json-schema.org/draft/2020-12/schema"}) is False
-    assert _is_any_type_schema({"enum": ["a", "b"]}) is False
-
-
-def test_add_object_type_preserves_any_type_schema():
-    """Test add_object_type does NOT add type:object to empty schemas (any type)."""
-    from litellm.llms.vertex_ai.common_utils import add_object_type
-
-    # Empty schema should be preserved (any type)
-    schema = {}
-    add_object_type(schema)
-    assert "type" not in schema, "Empty schema (any type) should not get type: object"
-
-    # Schema with only description should be preserved
-    schema = {"description": "Any JSON value"}
-    add_object_type(schema)
-    assert "type" not in schema
-
-    # Schema with $schema key should still get type: object (tool with no args)
-    schema = {"$schema": "https://json-schema.org/draft/2020-12/schema"}
-    add_object_type(schema)
-    assert schema["type"] == "object"
-
-
-def test_convert_anyof_preserves_any_type_members():
-    """Test convert_anyof_null_to_nullable does NOT coerce empty anyOf members to object."""
-    from litellm.llms.vertex_ai.common_utils import convert_anyof_null_to_nullable
-
-    # anyOf with empty schema and null — empty should be preserved
-    schema = {
-        "anyOf": [
-            {},
-            {"type": "null"},
-        ]
-    }
-    convert_anyof_null_to_nullable(schema)
-    # null should be removed, empty schema should be preserved (not coerced to object)
-    assert len(schema["anyOf"]) == 1
-    assert "type" not in schema["anyOf"][0] or schema["anyOf"][0].get("type") != "object"
-    assert schema["anyOf"][0].get("nullable") is True
-
-
-def test_build_vertex_schema_jsonvalue():
-    """
-    End-to-end: Pydantic JsonValue generates {} in $defs.
-    _build_vertex_schema should preserve any-type semantics.
-    Regression test for https://github.com/BerriAI/litellm/issues/22391
-    """
-    from litellm.llms.vertex_ai.common_utils import _build_vertex_schema
-
-    # Simulates what Pydantic generates for a model with JsonValue field
-    schema = {
-        "type": "object",
-        "properties": {
-            "name": {"type": "string"},
-            "value": {},  # after $ref resolution, this is what JsonValue becomes
-        },
-        "required": ["name", "value"],
-    }
-    result = _build_vertex_schema(schema)
-
-    # The "value" field should NOT have been coerced to type: object
-    value_schema = result["properties"]["value"]
-    assert value_schema.get("type") != "object", (
-        "JsonValue schema {} should not be coerced to {type: object}"
-    )

tests/test_litellm/llms/watsonx/test_watsonx.py

@@ -207,13 +207,11 @@ def test_watsonx_completion_regular_model_includes_model_id(
     assert "project_id" in json_data
 
 
-@pytest.mark.asyncio
-@pytest.mark.xdist_group("watsonx_heavy")
-async def test_watsonx_gpt_oss_prompt_transformation(monkeypatch):  # noqa: PLR0915
+def test_watsonx_gpt_oss_prompt_transformation(monkeypatch):
     """
     Test that gpt-oss-120b model transforms messages to proper format instead of simple concatenation.
 
-    This test starts from litellm.acompletion and verifies what gets sent in the final POST request body.
+    This test calls litellm.completion (sync) and verifies what gets sent in the final POST request body.
     Input messages should be transformed using the HuggingFace chat template from openai/gpt-oss-120b,
     not just concatenated as "You are chatgpt Hi there".
     """
@@ -229,39 +227,12 @@ async def test_watsonx_gpt_oss_prompt_transformation(monkeypatch):  # noqa: PLR0
         {"role": "user", "content": "Hi there"},
     ]
 
-    # Mock the HTTP client
-    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
-
-    client = AsyncHTTPHandler()
-
-    # Mock the token call
-    mock_token_response = Mock()
-    mock_token_response.json.return_value = {
-        "access_token": "mock_access_token",
-        "expires_in": 3600,
-    }
-    mock_token_response.raise_for_status = Mock()
-
-    # Mock the completion call
-    mock_completion_response = Mock()
-    mock_completion_response.status_code = 200
-    mock_completion_response.json.return_value = {
-        "results": [
-            {
-                "generated_text": "Hello! How can I help you?",
-                "generated_token_count": 10,
-                "input_token_count": 5,
-                "stop_reason": "stop",  # Required field for response transformation
-            }
-        ],
-        "model_id": "openai/gpt-oss-120b",
-    }
+    client = HTTPHandler()
 
     # Mock HuggingFace template fetch to make test deterministic and avoid network flakiness.
     # The test verifies that prompt transformation occurs (not simple concatenation), not the exact
     # HuggingFace template format. Using a mock template that produces the correct format is sufficient.
-    from unittest.mock import patch
-
+    #
     # Mock template that produces gpt-oss-120b-like format.
     # Note: This is a simplified version of the actual template. The real template is more complex
     # (adds metadata, handles tools, thinking messages, etc.), but this captures the key aspects:
@@ -277,105 +248,46 @@ async def test_watsonx_gpt_oss_prompt_transformation(monkeypatch):  # noqa: PLR0
         },
     }
 
-    async def mock_aget_tokenizer_config(hf_model_name: str):
-        return mock_tokenizer_config
-
-    async def mock_aget_chat_template_file(hf_model_name: str):
-        # Return failure to use tokenizer_config instead
-        return {"status": "failure"}
-
-    # Set cached tokenizer config directly to avoid race conditions with parallel tests.
-    # When running with pytest-xdist (-n 16), another test might populate the cache between
-    # clearing it and the actual usage. By setting the cache directly, we ensure the correct
-    # template is always used regardless of test execution order.
+    # Isolate known_tokenizer_config so parallel tests don't interfere.
+    # monkeypatch.setitem restores the original value on teardown.
     hf_model = "openai/gpt-oss-120b"
-    litellm.known_tokenizer_config[hf_model] = mock_tokenizer_config
+    monkeypatch.setitem(litellm.known_tokenizer_config, hf_model, mock_tokenizer_config)
 
-    # Also create sync mock functions in case the fallback sync path is used
-    def mock_get_tokenizer_config(hf_model_name: str):
-        return mock_tokenizer_config
-
-    def mock_get_chat_template_file(hf_model_name: str):
-        return {"status": "failure"}
-
-    # Async mock function for client.post to properly handle async method mocking
-    async def mock_post_func(*args, **kwargs):
-        return mock_completion_response
-
-    # Mock the token generation response to avoid actual API call
-    mock_token_get_response = Mock()
-    mock_token_get_response.json.return_value = {
+    # Mock IAM token generation to avoid real HTTP calls.
+    mock_token_response = Mock()
+    mock_token_response.json.return_value = {
         "access_token": "mock_access_token",
         "expires_in": 3600,
     }
-    mock_token_get_response.raise_for_status = Mock()
+    mock_token_response.raise_for_status = Mock()
 
-    # Pre-populate the WatsonX IAM token cache to avoid any HTTP calls for token generation.
-    # This prevents parallel test interference with litellm.module_level_client.
-    from litellm.llms.watsonx.common_utils import iam_token_cache
-    iam_token_cache.set_cache(key="test_api_key", value="mock_access_token", ttl=3600)
-
-    with patch.object(client, "post", side_effect=mock_post_func) as mock_post, patch.object(
-        litellm.module_level_client, "post", return_value=mock_token_get_response
-    ), patch(
-        "litellm.litellm_core_utils.prompt_templates.huggingface_template_handler._aget_tokenizer_config",
-        side_effect=mock_aget_tokenizer_config,
-    ), patch(
-        "litellm.litellm_core_utils.prompt_templates.huggingface_template_handler._aget_chat_template_file",
-        side_effect=mock_aget_chat_template_file,
-    ), patch(
-        "litellm.litellm_core_utils.prompt_templates.huggingface_template_handler._get_tokenizer_config",
-        side_effect=mock_get_tokenizer_config,
-    ), patch(
-        "litellm.litellm_core_utils.prompt_templates.huggingface_template_handler._get_chat_template_file",
-        side_effect=mock_get_chat_template_file,
+    with patch.object(client, "post") as mock_post, patch.object(
+        litellm.module_level_client, "post", return_value=mock_token_response
     ):
         try:
-            # Call acompletion with messages
-            await litellm.acompletion(
+            completion(
                 model=model,
                 messages=messages,
                 api_key="test_api_key",
                 client=client,
             )
         except Exception as e:
-            # May fail due to incomplete mocking, but we should have captured the request
-            print(f"Exception (may be expected): {e}")
+            print(f"Caught expected exception: {e}")
 
     # Verify the POST was called
     assert (
-        mock_post.call_count >= 1
-    ), f"POST should have been called at least once, got {mock_post.call_count}"
+        mock_post.call_count == 1
+    ), f"POST should have been called exactly once, got {mock_post.call_count}"
 
-    # Get the request body from the first call
-    # Use call_args_list to be more robust - get the first call's arguments
-    assert len(mock_post.call_args_list) > 0, "mock_post should have at least one call"
-    call_args = mock_post.call_args_list[0]
-    assert call_args is not None, "call_args should not be None"
+    # Get the request body
+    call_args = mock_post.call_args
     assert "data" in call_args.kwargs, "call_args.kwargs should contain 'data'"
     json_data = json.loads(call_args.kwargs["data"])
 
-    print(f"\n{'='*80}")
-    print(f"Input messages to litellm.acompletion:")
-    print(json.dumps(messages, indent=2))
-    print(f"\n{'='*80}")
-    print(f"Final POST request body:")
-    print(json.dumps(json_data, indent=2))
-    print(f"{'='*80}\n")
-
     # Verify the transformed input is in the request
     assert "input" in json_data, "Request should have 'input' field"
     transformed_prompt = json_data["input"]
 
-    # Verify transformation occurred
-    assert transformed_prompt is not None, (
-        "Prompt transformation failed - the template should have been applied to transform "
-        "messages into the correct format for gpt-oss-120b."
-    )
-
-    print(f"Transformed prompt: {repr(transformed_prompt)}")
-    print(f"Prompt length: {len(transformed_prompt)}")
-
     # Verify it's NOT simple concatenation
     simple_concat = "You are chatgpt Hi there"
     assert transformed_prompt != simple_concat, (