From 11225b7f13e28dcf5e7f438b71edc0fb803319f4 Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Fri, 31 Jul 2026 18:48:31 +0000
Subject: [PATCH 01/67] fix(spend): sum multi-round session duration in logs UI

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../spend_management_endpoints.py             |  8 +++
 .../test_spend_management_endpoints.py        | 72 +++++++++++++++++++
 .../RequestLogsTableColumns.test.tsx          | 31 ++++++++
 .../view_logs/RequestLogsTableColumns.tsx     | 18 +++--
 .../src/components/view_logs/columns.tsx      |  1 +
 5 files changed, 125 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py
index 42788227acc..d49bd6eb912 100644
--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py
+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py
@@ -3461,6 +3461,12 @@ async def _build_ui_spend_logs_response(
                 """
                 SELECT session_id,
                        COALESCE(SUM(spend), 0)::double precision AS session_total_spend,
+                       COALESCE(SUM(
+                           COALESCE(
+                               request_duration_ms,
+                               (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER
+                           )
+                       ), 0)::double precision AS session_total_duration_ms,
                        COUNT(*) FILTER (
                            WHERE call_type IN ('call_mcp_tool', 'list_mcp_tools')
                        )::int AS mcp_tool_call_count,
@@ -3478,6 +3484,7 @@ async def _build_ui_spend_logs_response(
             session_spend_map = {
                 row["session_id"]: {
                     "session_total_spend": float(row.get("session_total_spend") or 0.0),
+                    "session_total_duration_ms": int(row.get("session_total_duration_ms") or 0),
                     "mcp_tool_call_count": int(row.get("mcp_tool_call_count") or 0),
                     "mcp_tool_call_spend": float(row.get("mcp_tool_call_spend") or 0.0),
                 }
@@ -3499,6 +3506,7 @@ async def _build_ui_spend_logs_response(
             session_stats = session_spend_map.get(sid) if sid else None
             if session_stats:
                 row_dict["session_total_spend"] = session_stats["session_total_spend"]
+                row_dict["session_total_duration_ms"] = session_stats["session_total_duration_ms"]
                 if session_stats["mcp_tool_call_count"]:
                     row_dict["mcp_tool_call_count"] = session_stats["mcp_tool_call_count"]
                     row_dict["mcp_tool_call_spend"] = session_stats["mcp_tool_call_spend"]
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
index aa20c3f6ed4..c93810bdce0 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
@@ -3516,6 +3516,78 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_spend():
     assert call_args[2] == [api_key]
 
 
+@pytest.mark.asyncio
+async def test_build_ui_spend_logs_response_sums_multi_round_session_duration():
+    """
+    Regression test: a multi-round session collapses into a single UI row, so that row
+    must carry the duration of every round summed, not just the representative call's.
+    Rows written before request_duration_ms existed are NULL, so the aggregate falls back
+    to endTime - startTime for them.
+    """
+    from litellm.proxy.spend_tracking.spend_management_endpoints import (
+        _build_ui_spend_logs_response,
+    )
+
+    session_id = "sess-multi-round-duration"
+    api_key = "hashed-key-xyz"
+    dict_rows = [
+        {
+            "request_id": "req-1",
+            "session_id": session_id,
+            "call_type": "completion",
+            "api_key": api_key,
+            "spend": 0.01,
+            "request_duration_ms": 1200,
+        },
+        {
+            "request_id": "req-2",
+            "session_id": session_id,
+            "call_type": "completion",
+            "api_key": api_key,
+            "spend": 0.02,
+            "request_duration_ms": 4200,
+        },
+    ]
+
+    mock_prisma = MagicMock()
+    mock_prisma.db.litellm_spendlogs.group_by = AsyncMock(
+        return_value=[{"session_id": session_id, "_count": {"session_id": 2}}]
+    )
+    mock_prisma.db.query_raw = AsyncMock(
+        return_value=[
+            {
+                "session_id": session_id,
+                "session_total_spend": 0.03,
+                "session_total_duration_ms": 5400.0,
+                "mcp_tool_call_count": 0,
+                "mcp_tool_call_spend": 0.0,
+            }
+        ]
+    )
+
+    result = await _build_ui_spend_logs_response(
+        prisma_client=mock_prisma,
+        data=dict_rows,
+        total_records=2,
+        page=1,
+        page_size=50,
+        total_pages=1,
+        enrich_session_counts=True,
+    )
+
+    rows = result["data"]
+    assert [row["session_total_duration_ms"] for row in rows] == [5400, 5400]
+    assert all(isinstance(row["session_total_duration_ms"], int) for row in rows)
+    assert [row["request_duration_ms"] for row in rows] == [1200, 4200]
+
+    _, call_args, _ = mock_prisma.db.query_raw.mock_calls[0]
+    sql = " ".join(call_args[0].split())
+    assert (
+        'SUM( COALESCE( request_duration_ms, (EXTRACT(EPOCH FROM ("endTime" - "startTime")) * 1000)::INTEGER ) )'
+        in sql
+    )
+
+
 # ---------------------------------------------------------------------------
 # Tests for /spend/logs team-member permission
 # ---------------------------------------------------------------------------
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
index 2fdc8455ca1..f099edd249a 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
@@ -75,6 +75,37 @@ describe("Cost column", () => {
   });
 });
 
+describe("Duration column", () => {
+  it("shows the summed session duration, not the representative call's duration, for a multi-round session", () => {
+    renderRows([
+      logEntry({
+        request_id: "req-session-duration",
+        request_duration_ms: 1200,
+        session_id: "sess-1",
+        session_total_count: 3,
+        session_total_duration_ms: 5400,
+      }),
+    ]);
+
+    expect(screen.getByText("5.40")).toBeInTheDocument();
+    expect(screen.queryByText("1.20")).not.toBeInTheDocument();
+  });
+
+  it("shows the call's own duration for a single-call session", () => {
+    renderRows([
+      logEntry({
+        request_id: "req-single-duration",
+        request_duration_ms: 1200,
+        session_id: "sess-2",
+        session_total_count: 1,
+        session_total_duration_ms: 1200,
+      }),
+    ]);
+
+    expect(screen.getByText("1.20")).toBeInTheDocument();
+  });
+});
+
 describe("row action cells", () => {
   it("reports the key hash through the injected dependency rather than a row field", async () => {
     const user = userEvent.setup();
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
index 4e5a83ac7dd..ffdd16e26a9 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
@@ -160,13 +160,21 @@ export const getRequestLogsTableColumns = ({
     enableSorting: true,
     meta: { numeric: true },
     cell: ({ row }) => {
-      const ms = row.original.request_duration_ms;
+      const log = row.original;
+      const isMultiCallSession = (log.session_total_count || 1) > 1;
+      const ms =
+        isMultiCallSession && log.session_total_duration_ms != null
+          ? log.session_total_duration_ms
+          : log.request_duration_ms;
       if (ms == null) return <span>-</span>;
       return (
-        <CellTooltip
-          content={`${ms}ms`}
-          trigger={<span className="max-w-[15ch] truncate inline-block">{(ms / 1000).toFixed(2)}</span>}
-        />
+        <div className="flex flex-col items-end">
+          <CellTooltip
+            content={`${ms}ms`}
+            trigger={<span className="max-w-[15ch] truncate inline-block">{(ms / 1000).toFixed(2)}</span>}
+          />
+          {isMultiCallSession && <span className="text-[10px] text-gray-400">session total</span>}
+        </div>
       );
     },
   },
diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
index d4f784bf165..5b89c6f282c 100644
--- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
@@ -42,6 +42,7 @@ export type LogEntry = {
   request_duration_ms?: number;
   session_total_count?: number;
   session_total_spend?: number;
+  session_total_duration_ms?: number;
   mcp_tool_call_count?: number;
   mcp_tool_call_spend?: number;
   session_llm_count?: number;

From c4ab243fa63e76423d44ce3efa91c641c170a6e3 Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Fri, 31 Jul 2026 19:25:57 +0000
Subject: [PATCH 02/67] fix(ui): only label logs cost and duration a session
 total when the aggregate exists

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../RequestLogsTableColumns.test.tsx          | 29 +++++++++++++++++++
 .../view_logs/RequestLogsTableColumns.tsx     | 13 ++++-----
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
index f099edd249a..0b54f78736d 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
@@ -73,6 +73,20 @@ describe("Cost column", () => {
     expect(screen.queryByText("$0.010000")).not.toBeInTheDocument();
     expect(screen.getByText("session total")).toBeInTheDocument();
   });
+
+  it("does not label the per-call spend a session total when the aggregate is unavailable", () => {
+    renderRows([
+      logEntry({
+        request_id: "req-session-no-aggregate",
+        spend: 0.01,
+        session_id: "sess-1",
+        session_total_count: 3,
+      }),
+    ]);
+
+    expect(screen.getByText("$0.010000")).toBeInTheDocument();
+    expect(screen.queryByText("session total")).not.toBeInTheDocument();
+  });
 });
 
 describe("Duration column", () => {
@@ -89,6 +103,21 @@ describe("Duration column", () => {
 
     expect(screen.getByText("5.40")).toBeInTheDocument();
     expect(screen.queryByText("1.20")).not.toBeInTheDocument();
+    expect(screen.getByText("session total")).toBeInTheDocument();
+  });
+
+  it("does not label the per-call duration a session total when the aggregate is unavailable", () => {
+    renderRows([
+      logEntry({
+        request_id: "req-no-aggregate",
+        request_duration_ms: 1200,
+        session_id: "sess-3",
+        session_total_count: 3,
+      }),
+    ]);
+
+    expect(screen.getByText("1.20")).toBeInTheDocument();
+    expect(screen.queryByText("session total")).not.toBeInTheDocument();
   });
 
   it("shows the call's own duration for a single-call session", () => {
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
index ffdd16e26a9..1bd7c5c8dba 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
@@ -133,7 +133,8 @@ export const getRequestLogsTableColumns = ({
       const mcpCount = log.mcp_tool_call_count || 0;
       const mcpSpend = log.mcp_tool_call_spend || 0;
       const isMultiCallSession = (log.session_total_count || 1) > 1;
-      const spend = isMultiCallSession && log.session_total_spend != null ? log.session_total_spend : log.spend;
+      const sessionTotalSpend = isMultiCallSession ? log.session_total_spend : undefined;
+      const spend = sessionTotalSpend ?? log.spend;
       const money = (
         <span>
           <MoneyCell value={spend} decimals={6} />
@@ -143,7 +144,7 @@ export const getRequestLogsTableColumns = ({
       return (
         <div className="flex flex-col items-end">
           {spend ? <CellTooltip content={`$${String(spend)}`} trigger={money} /> : money}
-          {isMultiCallSession && <span className="text-[10px] text-gray-400">session total</span>}
+          {sessionTotalSpend != null && <span className="text-[10px] text-gray-400">session total</span>}
           {mcpCount > 0 && mcpSpend > 0 && (
             <span className="text-[10px] text-amber-600">
               incl. {getSpendString(mcpSpend)} from {mcpCount} MCP
@@ -162,10 +163,8 @@ export const getRequestLogsTableColumns = ({
     cell: ({ row }) => {
       const log = row.original;
       const isMultiCallSession = (log.session_total_count || 1) > 1;
-      const ms =
-        isMultiCallSession && log.session_total_duration_ms != null
-          ? log.session_total_duration_ms
-          : log.request_duration_ms;
+      const sessionTotalMs = isMultiCallSession ? log.session_total_duration_ms : undefined;
+      const ms = sessionTotalMs ?? log.request_duration_ms;
       if (ms == null) return <span>-</span>;
       return (
         <div className="flex flex-col items-end">
@@ -173,7 +172,7 @@ export const getRequestLogsTableColumns = ({
             content={`${ms}ms`}
             trigger={<span className="max-w-[15ch] truncate inline-block">{(ms / 1000).toFixed(2)}</span>}
           />
-          {isMultiCallSession && <span className="text-[10px] text-gray-400">session total</span>}
+          {sessionTotalMs != null && <span className="text-[10px] text-gray-400">session total</span>}
         </div>
       );
     },

From 003b53abbb8ad98bccddc47c0fd54c6cb4d461a2 Mon Sep 17 00:00:00 2001
From: jesus <jesus@berri.ai>
Date: Wed, 9 Sep 2026 22:03:04 +0000
Subject: [PATCH 03/67] feat(cli): sync Codex /model picker from proxy
 /v1/models in lite codex

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/client/cli/commands/agents.py   | 180 ++++++++++--
 .../cli/commands/codex_base_instructions.md   | 275 ++++++++++++++++++
 pyproject.toml                                |   1 +
 .../proxy/client/cli/test_agents.py           | 176 ++++++++++-
 4 files changed, 602 insertions(+), 30 deletions(-)
 create mode 100644 litellm/proxy/client/cli/commands/codex_base_instructions.md

diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py
index ea1eed65505..67d2d96e9d6 100644
--- a/litellm/proxy/client/cli/commands/agents.py
+++ b/litellm/proxy/client/cli/commands/agents.py
@@ -8,7 +8,7 @@ from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
 from types import MappingProxyType
-from typing import Final, TypeAlias
+from typing import Final, Literal, TypeAlias
 
 import click
 import requests
@@ -65,6 +65,9 @@ _INSTALL_DOCS: Final[dict[str, str]] = {
 _HIDDEN_AGENTS: Final = frozenset({"pi"})
 
 CODEX_PROXY_PROVIDER: Final = "litellm"
+CODEX_HOME_ENV: Final = "CODEX_HOME"
+CODEX_MODEL_CATALOG_FILENAME: Final = "litellm-models.json"
+_CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
 
 
 class AgentRunError(Exception):
@@ -242,7 +245,7 @@ def agent_launch_args(command: str, base_url: str) -> list[str]:
 
 
 class ListedModel(BaseModel):
-    """The fields of a /v1/models entry that an OpenCode model entry is built from."""
+    """The fields of a /v1/models entry that an OpenCode or Codex model entry is built from."""
 
     id: str
     mode: str | None = None
@@ -255,7 +258,7 @@ class _ModelListing(BaseModel):
 
 
 _MODEL_LISTING: Final = TypeAdapter(_ModelListing)
-_OPENCODE_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
+_CHAT_MODES: Final[frozenset[str]] = frozenset({"chat", "responses"})
 _NO_EXTRA_ENV: Final[Mapping[str, str]] = MappingProxyType({})
 
 
@@ -264,6 +267,40 @@ class ModelSyncSkipped:
     reason: str
 
 
+@dataclass(frozen=True, slots=True)
+class ModelSyncArgs:
+    """CLI args, placed before the user's own, that hand an agent the synced model list."""
+
+    args: tuple[str, ...]
+
+
+ModelSyncResult: TypeAlias = Mapping[str, str] | ModelSyncArgs | ModelSyncSkipped
+
+
+def _chat_models(models: Sequence[ListedModel]) -> tuple[ListedModel, ...]:
+    return tuple(m for m in models if m.mode is None or m.mode in _CHAT_MODES)
+
+
+def _fetch_model_listing(
+    base_url: str,
+    api_key: str,
+    *,
+    get: Callable[..., requests.Response],
+) -> tuple[ListedModel, ...] | ModelSyncSkipped:
+    url: Final = base_url.rstrip("/") + "/v1/models"
+    try:
+        resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
+    except requests.RequestException as e:
+        return ModelSyncSkipped(f"could not reach {url}: {e}")
+    if resp.status_code != 200:
+        return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}")
+    try:
+        listing: Final = _MODEL_LISTING.validate_json(resp.content)
+    except ValidationError:
+        return ModelSyncSkipped(f"{url} returned an unexpected body")
+    return listing.data
+
+
 class _OpenCodeLimit(BaseModel):
     context: int
     output: int
@@ -307,7 +344,7 @@ def opencode_provider_config(base_url: str, models: Sequence[ListedModel]) -> st
     it never lands in the config text. OpenCode merges this inline config over
     the user's own files, leaving unrelated keys and providers untouched.
     """
-    chat_models: Final = tuple(m for m in models if m.mode is None or m.mode in _OPENCODE_CHAT_MODES)
+    chat_models: Final = _chat_models(models)
     provider: Final = _OpenCodeProvider(
         npm=OPENCODE_PROVIDER_NPM,
         name=OPENCODE_PROVIDER_NAME,
@@ -337,18 +374,109 @@ def opencode_model_sync_env(
     """
     if OPENCODE_CONFIG_CONTENT_ENV in base_env:
         return ModelSyncSkipped(f"{OPENCODE_CONFIG_CONTENT_ENV} is already set")
-    url: Final = base_url.rstrip("/") + "/v1/models"
+    listing: Final = _fetch_model_listing(base_url, api_key, get=get)
+    if isinstance(listing, ModelSyncSkipped):
+        return listing
+    return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
+
+
+class _CodexTruncationPolicy(BaseModel):
+    mode: Literal["bytes"] = "bytes"
+    limit: int = 10_000
+
+
+class _CodexModel(BaseModel):
+    """One `ModelInfo` entry of a Codex model catalog.
+
+    Every field Codex's deserializer has no default for is spelled out here; the
+    values match the fallback metadata Codex uses today for a model slug it
+    does not know, so picking a proxy model behaves the same as `codex -m` did.
+    """
+
+    slug: str
+    display_name: str
+    description: None = None
+    supported_reasoning_levels: tuple[()] = ()
+    shell_type: Literal["unified_exec"] = "unified_exec"
+    visibility: Literal["list"] = "list"
+    supported_in_api: Literal[True] = True
+    priority: int
+    availability_nux: None = None
+    upgrade: None = None
+    support_verbosity: Literal[False] = False
+    default_verbosity: None = None
+    apply_patch_tool_type: None = None
+    truncation_policy: _CodexTruncationPolicy = _CodexTruncationPolicy()
+    experimental_supported_tools: tuple[()] = ()
+    context_window: int | None
+    base_instructions: str
+
+
+class _CodexCatalog(BaseModel):
+    models: tuple[_CodexModel, ...]
+
+
+def codex_model_catalog(models: Sequence[ListedModel]) -> str | None:
+    """The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
+
+    Codex refuses an empty catalog, hence None instead of `{"models": []}`.
+    Passing a catalog replaces Codex's built-in one, so every entry carries the
+    same base instructions Codex itself uses, otherwise the agent would run
+    without a system prompt.
+    """
+    chat_models: Final = _chat_models(models)
+    if not chat_models:
+        return None
+    instructions: Final = _CODEX_BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8")
+    catalog: Final = _CodexCatalog(
+        models=tuple(
+            _CodexModel(
+                slug=m.id,
+                display_name=m.id,
+                priority=index,
+                context_window=m.max_input_tokens,
+                base_instructions=instructions,
+            )
+            for index, m in enumerate(chat_models)
+        )
+    )
+    return catalog.model_dump_json()
+
+
+def codex_model_catalog_path(env: Mapping[str, str]) -> Path:
+    override: Final = env.get(CODEX_HOME_ENV)
+    root: Final = Path(override) if override else Path.home() / ".codex"
+    return root / CODEX_MODEL_CATALOG_FILENAME
+
+
+def codex_model_sync_args(
+    base_env: Mapping[str, str],
+    base_url: str,
+    api_key: str,
+    *,
+    get: Callable[..., requests.Response] = requests.get,
+) -> ModelSyncArgs | ModelSyncSkipped:
+    """`-c model_catalog_json=...` pointing Codex at the proxy's model list, or why it was skipped.
+
+    Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
+    must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
+    rewritten on every launch. The key never lands in the file. A failed fetch
+    or write is reported rather than raised: Codex still launches with its
+    built-in catalog and takes a proxy model by name via -m.
+    """
+    listing: Final = _fetch_model_listing(base_url, api_key, get=get)
+    if isinstance(listing, ModelSyncSkipped):
+        return listing
+    catalog: Final = codex_model_catalog(listing)
+    if catalog is None:
+        return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
+    path: Final = codex_model_catalog_path(base_env)
     try:
-        resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
-    except requests.RequestException as e:
-        return ModelSyncSkipped(f"could not reach {url}: {e}")
-    if resp.status_code != 200:
-        return ModelSyncSkipped(f"{url} returned HTTP {resp.status_code}")
-    try:
-        listing: Final = _MODEL_LISTING.validate_json(resp.content)
-    except ValidationError:
-        return ModelSyncSkipped(f"{url} returned an unexpected body")
-    return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing.data)})
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(catalog, encoding="utf-8")
+    except OSError as e:
+        return ModelSyncSkipped(f"could not write {path}: {e}")
+    return ModelSyncArgs(("-c", f"model_catalog_json={json.dumps(str(path))}"))
 
 
 def agent_model_sync_env(
@@ -359,18 +487,21 @@ def agent_model_sync_env(
     skip_verify: bool,
     *,
     get: Callable[..., requests.Response] = requests.get,
-) -> Mapping[str, str] | ModelSyncSkipped:
-    """Extra env an agent needs to see the proxy's model list.
+) -> ModelSyncResult:
+    """Extra env or args an agent needs to see the proxy's model list.
 
-    Only OpenCode needs one: Claude Code discovers models through
-    CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY and Codex takes the model by name.
+    OpenCode takes it as env, Codex as a `-c` override; Claude Code discovers
+    models itself through CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY.
     skip_verify means the caller wants no pre-launch proxy call at all, so the
     listing is skipped too rather than hanging on an offline proxy.
     """
-    if os.path.basename(command) != "opencode":
+    agent: Final = os.path.basename(command)
+    if agent not in ("opencode", "codex"):
         return _NO_EXTRA_ENV
     if skip_verify:
         return ModelSyncSkipped(f"{_SKIP_VERIFY_FLAG} was passed")
+    if agent == "codex":
+        return codex_model_sync_args(base_env, base_url, api_key, get=get)
     return opencode_model_sync_env(base_env, base_url, api_key, get=get)
 
 
@@ -498,9 +629,7 @@ def run_agent(
     base_env: Mapping[str, str] | None = None,
     which: Callable[[str], str | None] = shutil.which,
     verify: Callable[[str, str], None] = verify_proxy_key,
-    sync_models: Callable[[str, Mapping[str, str], str, str, bool], Mapping[str, str] | ModelSyncSkipped] = (
-        agent_model_sync_env
-    ),
+    sync_models: Callable[[str, Mapping[str, str], str, str, bool], ModelSyncResult] = agent_model_sync_env,
     warn: Callable[[str], None] = _warn,
     launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _hand_off,
     reattach_terminal: Callable[[], None] | None = None,
@@ -537,10 +666,11 @@ def run_agent(
     env: Final = MappingProxyType(
         {
             **build_agent_env(env_before_sync, base_url, api_key, profiles),
-            **(_NO_EXTRA_ENV if isinstance(synced, ModelSyncSkipped) else synced),
+            **(synced if isinstance(synced, Mapping) else _NO_EXTRA_ENV),
         }
     )
-    extra_args: Final = (*agent_launch_args(command[0], base_url), *prepared_args)
+    synced_args: Final = synced.args if isinstance(synced, ModelSyncArgs) else ()
+    extra_args: Final = (*agent_launch_args(command[0], base_url), *synced_args, *prepared_args)
     if reattach_terminal is not None:
         reattach_terminal()
     launcher(binary, [command[0], *extra_args, *command[1:]], env)
diff --git a/litellm/proxy/client/cli/commands/codex_base_instructions.md b/litellm/proxy/client/cli/commands/codex_base_instructions.md
new file mode 100644
index 00000000000..907ff8b8770
--- /dev/null
+++ b/litellm/proxy/client/cli/commands/codex_base_instructions.md
@@ -0,0 +1,275 @@
+You are a coding agent running in the Codex CLI, a terminal-based coding assistant. Codex CLI is an open source project led by OpenAI. You are expected to be precise, safe, and helpful.
+
+Your capabilities:
+
+- Receive user prompts and other context provided by the harness, such as files in the workspace.
+- Communicate with the user by streaming thinking & responses, and by making & updating plans.
+- Emit function calls to run terminal commands and apply patches. Depending on how this specific run is configured, you can request that these function calls be escalated to the user for approval before running. More on this in the "Sandbox and approvals" section.
+
+Within this context, Codex refers to the open-source agentic coding interface (not the old Codex language model built by OpenAI).
+
+# How you work
+
+## Personality
+
+Your default personality and tone is concise, direct, and friendly. You communicate efficiently, always keeping the user clearly informed about ongoing actions without unnecessary detail. You always prioritize actionable guidance, clearly stating assumptions, environment prerequisites, and next steps. Unless explicitly asked, you avoid excessively verbose explanations about your work.
+
+# AGENTS.md spec
+- Repos often contain AGENTS.md files. These files can appear anywhere within the repository.
+- These files are a way for humans to give you (the agent) instructions or tips for working within the container.
+- Some examples might be: coding conventions, info about how code is organized, or instructions for how to run or test code.
+- Instructions in AGENTS.md files:
+    - The scope of an AGENTS.md file is the entire directory tree rooted at the folder that contains it.
+    - For every file you touch in the final patch, you must obey instructions in any AGENTS.md file whose scope includes that file.
+    - Instructions about code style, structure, naming, etc. apply only to code within the AGENTS.md file's scope, unless the file states otherwise.
+    - More-deeply-nested AGENTS.md files take precedence in the case of conflicting instructions.
+    - Direct system/developer/user instructions (as part of a prompt) take precedence over AGENTS.md instructions.
+- The contents of the AGENTS.md file at the root of the repo and any directories from the CWD up to the root are included with the developer message and don't need to be re-read. When working in a subdirectory of CWD, or a directory outside the CWD, check for any AGENTS.md files that may be applicable.
+
+## Responsiveness
+
+### Preamble messages
+
+Before making tool calls, send a brief preamble to the user explaining what you’re about to do. When sending preamble messages, follow these principles and examples:
+
+- **Logically group related actions**: if you’re about to run several related commands, describe them together in one preamble rather than sending a separate note for each.
+- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words for quick updates).
+- **Build on prior context**: if this is not your first tool call, use the preamble message to connect the dots with what’s been done so far and create a sense of momentum and clarity for the user to understand your next actions.
+- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.
+- **Exception**: Avoid adding a preamble for every trivial read (e.g., `cat` a single file) unless it’s part of a larger grouped action.
+
+**Examples:**
+
+- “I’ve explored the repo; now checking the API route definitions.”
+- “Next, I’ll patch the config and update the related tests.”
+- “I’m about to scaffold the CLI commands and helper functions.”
+- “Ok cool, so I’ve wrapped my head around the repo. Now digging into the API routes.”
+- “Config’s looking tidy. Next up is patching helpers to keep things in sync.”
+- “Finished poking at the DB gateway. I will now chase down error handling.”
+- “Alright, build pipeline order is interesting. Checking how it reports failures.”
+- “Spotted a clever caching util; now hunting where it gets used.”
+
+## Planning
+
+You have access to an `update_plan` tool which tracks steps and progress and renders them to the user. Using the tool helps demonstrate that you've understood the task and convey how you're approaching it. Plans can help to make complex, ambiguous, or multi-phase work clearer and more collaborative for the user. A good plan should break the task into meaningful, logically ordered steps that are easy to verify as you go.
+
+Note that plans are not for padding out simple work with filler steps or stating the obvious. The content of your plan should not involve doing anything that you aren't capable of doing (i.e. don't try to test things that you can't test). Do not use plans for simple or single-step queries that you can just do or answer immediately.
+
+Do not repeat the full contents of the plan after an `update_plan` call — the harness already displays it. Instead, summarize the change made and highlight any important context or next step.
+
+Before running a command, consider whether or not you have completed the previous step, and make sure to mark it as completed before moving on to the next step. It may be the case that you complete all steps in your plan after a single pass of implementation. If this is the case, you can simply mark all the planned steps as completed. Sometimes, you may need to change plans in the middle of a task: call `update_plan` with the updated plan and make sure to provide an `explanation` of the rationale when doing so.
+
+Use a plan when:
+
+- The task is non-trivial and will require multiple actions over a long time horizon.
+- There are logical phases or dependencies where sequencing matters.
+- The work has ambiguity that benefits from outlining high-level goals.
+- You want intermediate checkpoints for feedback and validation.
+- When the user asked you to do more than one thing in a single prompt
+- The user has asked you to use the plan tool (aka "TODOs")
+- You generate additional steps while working, and plan to do them before yielding to the user
+
+### Examples
+
+**High-quality plans**
+
+Example 1:
+
+1. Add CLI entry with file args
+2. Parse Markdown via CommonMark library
+3. Apply semantic HTML template
+4. Handle code blocks, images, links
+5. Add error handling for invalid files
+
+Example 2:
+
+1. Define CSS variables for colors
+2. Add toggle with localStorage state
+3. Refactor components to use variables
+4. Verify all views for readability
+5. Add smooth theme-change transition
+
+Example 3:
+
+1. Set up Node.js + WebSocket server
+2. Add join/leave broadcast events
+3. Implement messaging with timestamps
+4. Add usernames + mention highlighting
+5. Persist messages in lightweight DB
+6. Add typing indicators + unread count
+
+**Low-quality plans**
+
+Example 1:
+
+1. Create CLI tool
+2. Add Markdown parser
+3. Convert to HTML
+
+Example 2:
+
+1. Add dark mode toggle
+2. Save preference
+3. Make styles look good
+
+Example 3:
+
+1. Create single-file HTML game
+2. Run quick sanity check
+3. Summarize usage instructions
+
+If you need to write a plan, only write high quality plans, not low quality ones.
+
+## Task execution
+
+You are a coding agent. Please keep going until the query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved. Autonomously resolve the query to the best of your ability, using the tools available to you, before coming back to the user. Do NOT guess or make up an answer.
+
+You MUST adhere to the following criteria when solving queries:
+
+- Working on the repo(s) in the current environment is allowed, even if they are proprietary.
+- Analyzing code for vulnerabilities is allowed.
+- Showing user code and tool call details is allowed.
+- Use the `apply_patch` tool to edit files (NEVER try `applypatch` or `apply-patch`, only `apply_patch`): {"command":["apply_patch","*** Begin Patch\\n*** Update File: path/to/file.py\\n@@ def example():\\n- pass\\n+ return 123\\n*** End Patch"]}
+
+If completing the user's task requires writing or modifying files, your code and final answer should follow these coding guidelines, though user instructions (i.e. AGENTS.md) may override these guidelines:
+
+- Fix the problem at the root cause rather than applying surface-level patches, when possible.
+- Avoid unneeded complexity in your solution.
+- Do not attempt to fix unrelated bugs or broken tests. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)
+- Update documentation as necessary.
+- Keep changes consistent with the style of the existing codebase. Changes should be minimal and focused on the task.
+- Use `git log` and `git blame` to search the history of the codebase if additional context is required.
+- NEVER add copyright or license headers unless specifically requested.
+- Do not waste tokens by re-reading files after calling `apply_patch` on them. The tool call will fail if it didn't work. The same goes for making folders, deleting folders, etc.
+- Do not `git commit` your changes or create new git branches unless explicitly requested.
+- Do not add inline comments within code unless explicitly requested.
+- Do not use one-letter variable names unless explicitly requested.
+- NEVER output inline citations like "【F:README.md†L5-L14】" in your outputs. The CLI is not able to render these so they will just be broken in the UI. Instead, if you output valid filepaths, users will be able to click on them to open the files in their editor.
+
+## Validating your work
+
+If the codebase has tests or the ability to build or run, consider using them to verify that your work is complete. 
+
+When testing, your philosophy should be to start as specific as possible to the code you changed so that you can catch issues efficiently, then make your way to broader tests as you build confidence. If there's no test for the code you changed, and if the adjacent patterns in the codebases show that there's a logical place for you to add a test, you may do so. However, do not add tests to codebases with no tests.
+
+Similarly, once you're confident in correctness, you can suggest or use formatting commands to ensure that your code is well formatted. If there are issues you can iterate up to 3 times to get formatting right, but if you still can't manage it's better to save the user time and present them a correct solution where you call out the formatting in your final message. If the codebase does not have a formatter configured, do not add one.
+
+For all of testing, running, building, and formatting, do not attempt to fix unrelated bugs. It is not your responsibility to fix them. (You may mention them to the user in your final message though.)
+
+Be mindful of whether to run validation commands proactively. In the absence of behavioral guidance:
+
+- When running in the non-interactive approval mode **never**, proactively run tests, lint and do whatever you need to ensure you've completed the task.
+- When working in interactive approval modes like **untrusted**, or **on-request**, hold off on running tests or lint commands until the user is ready for you to finalize your output, because these commands take time to run and slow down iteration. Instead suggest what you want to do next, and let the user confirm first.
+- When working on test-related tasks, such as adding tests, fixing tests, or reproducing a bug to verify behavior, you may proactively run tests regardless of approval mode. Use your judgement to decide whether this is a test-related task.
+
+## Ambition vs. precision
+
+For tasks that have no prior context (i.e. the user is starting something brand new), you should feel free to be ambitious and demonstrate creativity with your implementation.
+
+If you're operating in an existing codebase, you should make sure you do exactly what the user asks with surgical precision. Treat the surrounding codebase with respect, and don't overstep (i.e. changing filenames or variables unnecessarily). You should balance being sufficiently ambitious and proactive when completing tasks of this nature.
+
+You should use judicious initiative to decide on the right level of detail and complexity to deliver based on the user's needs. This means showing good judgment that you're capable of doing the right extras without gold-plating. This might be demonstrated by high-value, creative touches when scope of the task is vague; while being surgical and targeted when scope is tightly specified.
+
+## Sharing progress updates
+
+For especially longer tasks that you work on (i.e. requiring many tool calls, or a plan with multiple steps), you should provide progress updates back to the user at reasonable intervals. These updates should be structured as a concise sentence or two (no more than 8-10 words long) recapping progress so far in plain language: this update demonstrates your understanding of what needs to be done, progress so far (i.e. files explores, subtasks complete), and where you're going next.
+
+Before doing large chunks of work that may incur latency as experienced by the user (i.e. writing a new file), you should send a concise message to the user with an update indicating what you're about to do to ensure they know what you're spending time on. Don't start editing or writing large files before informing the user what you are doing and why.
+
+The messages you send before tool calls should describe what is immediately about to be done next in very concise language. If there was previous work done, this preamble message should also include a note about the work done so far to bring the user along.
+
+## Presenting your work and final message
+
+Your final message should read naturally, like an update from a concise teammate. For casual conversation, brainstorming tasks, or quick questions from the user, respond in a friendly, conversational tone. You should ask questions, suggest ideas, and adapt to the user’s style. If you've finished a large amount of work, when describing what you've done to the user, you should follow the final answer formatting guidelines to communicate substantive changes. You don't need to add structured formatting for one-word answers, greetings, or purely conversational exchanges.
+
+You can skip heavy formatting for single, simple actions or confirmations. In these cases, respond in plain sentences with any relevant next step or quick option. Reserve multi-section structured responses for results that need grouping or explanation.
+
+The user is working on the same computer as you, and has access to your work. As such there's no need to show the full contents of large files you have already written unless the user explicitly asks for them. Similarly, if you've created or modified files using `apply_patch`, there's no need to tell users to "save the file" or "copy the code into a file"—just reference the file path.
+
+If there's something that you think you could help with as a logical next step, concisely ask the user if they want you to do so. Good examples of this are running tests, committing changes, or building out the next logical component. If there’s something that you couldn't do (even with approval) but that the user might want to do (such as verifying changes by running the app), include those instructions succinctly.
+
+Brevity is very important as a default. You should be very concise (i.e. no more than 10 lines), but can relax this requirement for tasks where additional detail and comprehensiveness is important for the user's understanding.
+
+### Final answer structure and style guidelines
+
+You are producing plain text that will later be styled by the CLI. Follow these rules exactly. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value.
+
+**Section Headers**
+
+- Use only when they improve clarity — they are not mandatory for every answer.
+- Choose descriptive names that fit the content
+- Keep headers short (1–3 words) and in `**Title Case**`. Always start headers with `**` and end with `**`
+- Leave no blank line before the first bullet under a header.
+- Section headers should only be used where they genuinely improve scanability; avoid fragmenting the answer.
+
+**Bullets**
+
+- Use `-` followed by a space for every bullet.
+- Merge related points when possible; avoid a bullet for every trivial detail.
+- Keep bullets to one line unless breaking for clarity is unavoidable.
+- Group into short lists (4–6 bullets) ordered by importance.
+- Use consistent keyword phrasing and formatting across sections.
+
+**Monospace**
+
+- Wrap all commands, file paths, env vars, and code identifiers in backticks (`` `...` ``).
+- Apply to inline examples and to bullet keywords if the keyword itself is a literal file/command.
+- Never mix monospace and bold markers; choose one based on whether it’s a keyword (`**`) or inline code/path (`` ` ``).
+
+**File References**
+When referencing files in your response, make sure to include the relevant start line and always follow the below rules:
+  * Use inline code to make file paths clickable.
+  * Each reference should have a stand alone path. Even if it's the same file.
+  * Accepted: absolute, workspace‑relative, a/ or b/ diff prefixes, or bare filename/suffix.
+  * Line/column (1‑based, optional): :line[:column] or #Lline[Ccolumn] (column defaults to 1).
+  * Do not use URIs like file://, vscode://, or https://.
+  * Do not provide range of lines
+  * Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\repo\project\main.rs:12:5
+
+**Structure**
+
+- Place related bullets together; don’t mix unrelated concepts in the same section.
+- Order sections from general → specific → supporting info.
+- For subsections (e.g., “Binaries” under “Rust Workspace”), introduce with a bolded keyword bullet, then list items under it.
+- Match structure to complexity:
+  - Multi-part or detailed results → use clear headers and grouped bullets.
+  - Simple results → minimal headers, possibly just a short list or paragraph.
+
+**Tone**
+
+- Keep the voice collaborative and natural, like a coding partner handing off work.
+- Be concise and factual — no filler or conversational commentary and avoid unnecessary repetition
+- Use present tense and active voice (e.g., “Runs tests” not “This will run tests”).
+- Keep descriptions self-contained; don’t refer to “above” or “below”.
+- Use parallel structure in lists for consistency.
+
+**Don’t**
+
+- Don’t use literal words “bold” or “monospace” in the content.
+- Don’t nest bullets or create deep hierarchies.
+- Don’t output ANSI escape codes directly — the CLI renderer applies them.
+- Don’t cram unrelated keywords into a single bullet; split for clarity.
+- Don’t let keyword lists run long — wrap or reformat for scanability.
+
+Generally, ensure your final answers adapt their shape and depth to the request. For example, answers to code explanations should have a precise, structured explanation with code references that answer the question directly. For tasks with a simple implementation, lead with the outcome and supplement only with what’s needed for clarity. Larger changes can be presented as a logical walkthrough of your approach, grouping related steps, explaining rationale where it adds value, and highlighting next actions to accelerate the user. Your answers should provide the right level of detail while being easily scannable.
+
+For casual greetings, acknowledgements, or other one-off conversational messages that are not delivering substantive information or structured results, respond naturally without section headers or bullet formatting.
+
+# Tool Guidelines
+
+## Shell commands
+
+When using the shell, you must adhere to the following guidelines:
+
+- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)
+- Do not use python scripts to attempt to output larger chunks of a file.
+
+## `update_plan`
+
+A tool named `update_plan` is available to you. You can use it to keep an up‑to‑date, step‑by‑step plan for the task.
+
+To create a new plan, call `update_plan` with a short list of 1‑sentence steps (no more than 5-7 words each) with a `status` for each step (`pending`, `in_progress`, or `completed`).
+
+When steps have been completed, use `update_plan` to mark each finished step as `completed` and the next step you are working on as `in_progress`. There should always be exactly one `in_progress` step until everything is done. You can mark multiple items as complete in a single `update_plan` call.
+
+If all steps are complete, ensure you call `update_plan` to mark all steps as `completed`.
diff --git a/pyproject.toml b/pyproject.toml
index 448451f7f93..29609ce5ca1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -287,6 +287,7 @@ editable-profile = "dev"
 include = [
     "litellm/proxy/_experimental/out/**",
     "litellm/router_strategy/complexity_router/artifacts/*.json",
+    "litellm/proxy/client/cli/commands/codex_base_instructions.md",
 ]
 exclude = [
     "litellm/proxy/enterprise",
diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py
index 7804435a60d..bb4b99506a2 100644
--- a/tests/test_litellm/proxy/client/cli/test_agents.py
+++ b/tests/test_litellm/proxy/client/cli/test_agents.py
@@ -9,10 +9,9 @@ import pytest
 import requests
 from click.testing import CliRunner
 
-
-
 from litellm.proxy.client.cli.commands.agents import (
     AgentRunError,
+    ModelSyncArgs,
     ModelSyncSkipped,
     _hand_off,
     _replace_process,
@@ -22,6 +21,7 @@ from litellm.proxy.client.cli.commands.agents import (
     agent_model_sync_env,
     agent_profile,
     build_agent_env,
+    codex_model_sync_args,
     opencode_model_sync_env,
     run_agent,
     verify_proxy_key,
@@ -333,10 +333,10 @@ class TestOpencodeModelSync:
         assert isinstance(result, ModelSyncSkipped)
         assert "unexpected body" in result.reason
 
-    @pytest.mark.parametrize("command", ["claude", "codex", "/usr/bin/claude"])
-    def test_only_opencode_syncs(self, command):
+    @pytest.mark.parametrize("command", ["claude", "pi", "/usr/bin/claude"])
+    def test_only_opencode_and_codex_sync(self, command):
         def boom(*a, **k):
-            raise AssertionError("no agent other than opencode should call the proxy")
+            raise AssertionError("no agent other than opencode or codex should call the proxy")
 
         assert agent_model_sync_env(command, {}, "http://localhost:4000", "sk-key", False, get=boom) == {}
 
@@ -365,7 +365,173 @@ class TestOpencodeModelSync:
         assert _default_of(opencode_model_sync_env, "get") is requests.get
 
 
+class TestCodexModelSync:
+    @staticmethod
+    def _listing(*models):
+        return {"object": "list", "data": list(models)}
+
+    @staticmethod
+    def _row(model_id, **extra):
+        return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra}
+
+    def _sync(self, listing, codex_home, base_url="http://localhost:4000/"):
+        captured = {}
+
+        def fake_get(url, headers, timeout):
+            captured["url"] = url
+            captured["headers"] = headers
+            return _FakeResponse(200, listing)
+
+        result = codex_model_sync_args({"CODEX_HOME": str(codex_home)}, base_url, "sk-key", get=fake_get)
+        return captured, result
+
+    @staticmethod
+    def _catalog_path(result):
+        assert isinstance(result, ModelSyncArgs)
+        flag, override = result.args
+        assert flag == "-c"
+        key, _, value = override.partition("=")
+        assert key == "model_catalog_json"
+        return json.loads(value)
+
+    def test_writes_catalog_under_codex_home_and_points_codex_at_it(self, tmp_path):
+        listing = self._listing(self._row("gpt-5.5", mode="chat"), self._row("claude-opus-4-7"))
+        captured, result = self._sync(listing, tmp_path / "codex")
+
+        assert captured["url"] == "http://localhost:4000/v1/models"
+        assert captured["headers"] == {"Authorization": "Bearer sk-key"}
+        path = self._catalog_path(result)
+        assert path == str(tmp_path / "codex" / "litellm-models.json")
+        text = (tmp_path / "codex" / "litellm-models.json").read_text()
+        assert "sk-key" not in text
+        catalog = json.loads(text)
+        assert [m["slug"] for m in catalog["models"]] == ["gpt-5.5", "claude-opus-4-7"]
+        assert [m["display_name"] for m in catalog["models"]] == ["gpt-5.5", "claude-opus-4-7"]
+        assert [m["priority"] for m in catalog["models"]] == [0, 1]
+
+    def test_every_entry_has_the_fields_codex_requires(self, tmp_path):
+        _, result = self._sync(self._listing(self._row("m")), tmp_path)
+        entry = json.loads((tmp_path / "litellm-models.json").read_text())["models"][0]
+
+        assert entry["visibility"] == "list"
+        assert entry["supported_in_api"] is True
+        assert entry["shell_type"] == "unified_exec"
+        assert entry["supported_reasoning_levels"] == []
+        assert entry["truncation_policy"] == {"mode": "bytes", "limit": 10000}
+        assert entry["experimental_supported_tools"] == []
+        assert entry["support_verbosity"] is False
+        for nullable in ("description", "availability_nux", "upgrade", "default_verbosity", "apply_patch_tool_type"):
+            assert nullable in entry and entry[nullable] is None
+        assert entry["base_instructions"].startswith("You are a coding agent running in the Codex CLI")
+
+    def test_context_window_comes_from_max_input_tokens(self, tmp_path):
+        listing = self._listing(self._row("big", max_input_tokens=400000), self._row("unknown"))
+        _, result = self._sync(listing, tmp_path)
+        models = {m["slug"]: m for m in json.loads((tmp_path / "litellm-models.json").read_text())["models"]}
+        assert models["big"]["context_window"] == 400000
+        assert models["unknown"]["context_window"] is None
+
+    def test_non_chat_models_are_left_out(self, tmp_path):
+        listing = self._listing(
+            self._row("chat", mode="chat"),
+            self._row("resp", mode="responses"),
+            self._row("embed", mode="embedding"),
+            self._row("img", mode="image_generation"),
+        )
+        self._sync(listing, tmp_path)
+        slugs = {m["slug"] for m in json.loads((tmp_path / "litellm-models.json").read_text())["models"]}
+        assert slugs == {"chat", "resp"}
+
+    def test_listing_without_chat_models_is_skipped_and_writes_nothing(self, tmp_path):
+        _, result = self._sync(self._listing(self._row("embed", mode="embedding")), tmp_path)
+        assert isinstance(result, ModelSyncSkipped)
+        assert "no chat models" in result.reason
+        assert not (tmp_path / "litellm-models.json").exists()
+
+    def test_catalog_is_rewritten_on_every_launch(self, tmp_path):
+        self._sync(self._listing(self._row("old")), tmp_path)
+        self._sync(self._listing(self._row("new")), tmp_path)
+        slugs = [m["slug"] for m in json.loads((tmp_path / "litellm-models.json").read_text())["models"]]
+        assert slugs == ["new"]
+
+    def test_defaults_to_dot_codex_in_home(self, tmp_path, monkeypatch):
+        monkeypatch.setattr("pathlib.Path.home", classmethod(lambda cls: tmp_path))
+        result = codex_model_sync_args(
+            {}, "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m")))
+        )
+        assert self._catalog_path(result) == str(tmp_path / ".codex" / "litellm-models.json")
+
+    def test_unwritable_catalog_path_is_reported_not_raised(self, tmp_path):
+        blocker = tmp_path / "file"
+        blocker.write_text("")
+        _, result = self._sync(self._listing(self._row("m")), blocker / "codex")
+        assert isinstance(result, ModelSyncSkipped)
+        assert "could not write" in result.reason
+
+    def test_unreachable_proxy_is_reported_not_raised(self, tmp_path):
+        def boom(*a, **k):
+            raise requests.ConnectionError("refused")
+
+        result = codex_model_sync_args({"CODEX_HOME": str(tmp_path)}, "http://localhost:4000", "sk-key", get=boom)
+        assert isinstance(result, ModelSyncSkipped)
+        assert "refused" in result.reason
+        assert not (tmp_path / "litellm-models.json").exists()
+
+    @pytest.mark.parametrize(
+        ("response", "reason"),
+        [(_FakeResponse(500), "HTTP 500"), (_FakeResponse(200, {"data": "nope"}), "unexpected body")],
+    )
+    def test_bad_response_is_reported(self, tmp_path, response, reason):
+        result = codex_model_sync_args(
+            {"CODEX_HOME": str(tmp_path)}, "http://localhost:4000", "sk-key", get=lambda *a, **k: response
+        )
+        assert isinstance(result, ModelSyncSkipped)
+        assert reason in result.reason
+
+    @pytest.mark.parametrize("command", ["codex", "/opt/bin/codex"])
+    def test_codex_syncs_through_the_agent_dispatch(self, tmp_path, command):
+        result = agent_model_sync_env(
+            command,
+            {"CODEX_HOME": str(tmp_path)},
+            "http://localhost:4000",
+            "sk-key",
+            False,
+            get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m"))),
+        )
+        assert self._catalog_path(result) == str(tmp_path / "litellm-models.json")
+
+    def test_skip_verify_keeps_the_launch_offline(self):
+        def boom(*a, **k):
+            raise AssertionError("--skip-verify must not touch the proxy")
+
+        result = agent_model_sync_env("codex", {}, "http://localhost:4000", "sk-key", True, get=boom)
+        assert isinstance(result, ModelSyncSkipped)
+        assert "--skip-verify" in result.reason
+
+    def test_default_http_client_is_requests_get(self):
+        assert _default_of(codex_model_sync_args, "get") is requests.get
+
+
 class TestRunAgent:
+    def test_synced_args_precede_user_args_and_follow_provider_overrides(self):
+        calls = {}
+        run_agent(
+            "http://localhost:4000",
+            "sk-key",
+            ["codex", "exec", "hi"],
+            base_env={},
+            sync_models=lambda *a: ModelSyncArgs(("-c", 'model_catalog_json="/tmp/c.json"')),
+            which=lambda name: "/usr/local/bin/codex",
+            verify=lambda *a: None,
+            launcher=lambda p, a, e: calls.update(args=tuple(a), env=dict(e)),
+        )
+        args = calls["args"]
+        assert args[-2:] == ("exec", "hi")
+        assert args[args.index('model_catalog_json="/tmp/c.json"') - 1] == "-c"
+        assert args.index('model_provider="litellm"') < args.index('model_catalog_json="/tmp/c.json"') < args.index("exec")
+        assert calls["env"]["OPENAI_API_KEY"] == "sk-key"
+        assert "model_catalog_json" not in json.dumps(calls["env"])
+
     def test_synced_model_config_reaches_the_agent_alongside_profile_env(self):
         calls = {}
         run_agent(

From d1653fa40dd534c03633707eb7c451421e9a5af2 Mon Sep 17 00:00:00 2001
From: jesus <jesus@berri.ai>
Date: Wed, 9 Sep 2026 22:29:09 +0000
Subject: [PATCH 04/67] fix(cli): replace Codex catalog atomically and skip
 sync on unreadable instructions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/client/cli/commands/agents.py   | 34 ++++++---
 .../proxy/client/cli/test_agents.py           | 72 ++++++++++---------
 2 files changed, 63 insertions(+), 43 deletions(-)

diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py
index 67d2d96e9d6..8e3698985af 100644
--- a/litellm/proxy/client/cli/commands/agents.py
+++ b/litellm/proxy/client/cli/commands/agents.py
@@ -4,6 +4,7 @@ import re
 import shutil
 import subprocess
 import sys
+import tempfile
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
@@ -416,7 +417,7 @@ class _CodexCatalog(BaseModel):
     models: tuple[_CodexModel, ...]
 
 
-def codex_model_catalog(models: Sequence[ListedModel]) -> str | None:
+def codex_model_catalog(models: Sequence[ListedModel], instructions: str) -> str | None:
     """The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
 
     Codex refuses an empty catalog, hence None instead of `{"models": []}`.
@@ -427,7 +428,6 @@ def codex_model_catalog(models: Sequence[ListedModel]) -> str | None:
     chat_models: Final = _chat_models(models)
     if not chat_models:
         return None
-    instructions: Final = _CODEX_BASE_INSTRUCTIONS_PATH.read_text(encoding="utf-8")
     catalog: Final = _CodexCatalog(
         models=tuple(
             _CodexModel(
@@ -443,37 +443,49 @@ def codex_model_catalog(models: Sequence[ListedModel]) -> str | None:
     return catalog.model_dump_json()
 
 
-def codex_model_catalog_path(env: Mapping[str, str]) -> Path:
+def codex_model_catalog_path(env: Mapping[str, str], *, home: Callable[[], Path] = Path.home) -> Path:
     override: Final = env.get(CODEX_HOME_ENV)
-    root: Final = Path(override) if override else Path.home() / ".codex"
+    root: Final = Path(override) if override else home() / ".codex"
     return root / CODEX_MODEL_CATALOG_FILENAME
 
 
+def _replace_file(path: Path, text: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
+        _ = tmp.write(text)
+    os.replace(tmp.name, path)
+
+
 def codex_model_sync_args(
     base_env: Mapping[str, str],
     base_url: str,
     api_key: str,
     *,
     get: Callable[..., requests.Response] = requests.get,
+    home: Callable[[], Path] = Path.home,
+    instructions_path: Path = _CODEX_BASE_INSTRUCTIONS_PATH,
 ) -> ModelSyncArgs | ModelSyncSkipped:
     """`-c model_catalog_json=...` pointing Codex at the proxy's model list, or why it was skipped.
 
     Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
     must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
-    rewritten on every launch. The key never lands in the file. A failed fetch
-    or write is reported rather than raised: Codex still launches with its
-    built-in catalog and takes a proxy model by name via -m.
+    atomically replaced on every launch. The key never lands in the file. A
+    failed fetch, read or write is reported rather than raised: Codex still
+    launches with its built-in catalog and takes a proxy model by name via -m.
     """
     listing: Final = _fetch_model_listing(base_url, api_key, get=get)
     if isinstance(listing, ModelSyncSkipped):
         return listing
-    catalog: Final = codex_model_catalog(listing)
+    try:
+        instructions: Final = instructions_path.read_text(encoding="utf-8")
+    except OSError as e:
+        return ModelSyncSkipped(f"could not read {instructions_path}: {e}")
+    catalog: Final = codex_model_catalog(listing, instructions)
     if catalog is None:
         return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
-    path: Final = codex_model_catalog_path(base_env)
+    path: Final = codex_model_catalog_path(base_env, home=home)
     try:
-        path.parent.mkdir(parents=True, exist_ok=True)
-        path.write_text(catalog, encoding="utf-8")
+        _replace_file(path, catalog)
     except OSError as e:
         return ModelSyncSkipped(f"could not write {path}: {e}")
     return ModelSyncArgs(("-c", f"model_catalog_json={json.dumps(str(path))}"))
diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py
index bb4b99506a2..75e42c3eaa0 100644
--- a/tests/test_litellm/proxy/client/cli/test_agents.py
+++ b/tests/test_litellm/proxy/client/cli/test_agents.py
@@ -2,6 +2,7 @@ import inspect
 import json
 import os
 import sys
+from pathlib import Path
 from unittest.mock import patch
 
 import click
@@ -90,9 +91,7 @@ class TestAgentProfile:
 
 class TestBuildAgentEnv:
     def test_anthropic_profile_uses_bare_root_and_bearer(self):
-        env = build_agent_env(
-            {}, "http://localhost:4000/", "sk-key", frozenset({"anthropic"})
-        )
+        env = build_agent_env({}, "http://localhost:4000/", "sk-key", frozenset({"anthropic"}))
         assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000"
         assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key"
         assert env["ENABLE_TOOL_SEARCH"] == "true"
@@ -128,9 +127,7 @@ class TestBuildAgentEnv:
         assert "ANTHROPIC_API_KEY" not in env
 
     def test_openai_profile_appends_v1(self):
-        env = build_agent_env(
-            {}, "http://localhost:4000/", "sk-key", frozenset({"openai"})
-        )
+        env = build_agent_env({}, "http://localhost:4000/", "sk-key", frozenset({"openai"}))
         assert env["OPENAI_BASE_URL"] == "http://localhost:4000/v1"
         assert env["OPENAI_API_KEY"] == "sk-key"
         assert "ANTHROPIC_BASE_URL" not in env
@@ -138,9 +135,7 @@ class TestBuildAgentEnv:
         assert "CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY" not in env
 
     def test_both_profiles_set_everything(self):
-        env = build_agent_env(
-            {}, "http://localhost:4000", "sk-key", frozenset({"anthropic", "openai"})
-        )
+        env = build_agent_env({}, "http://localhost:4000", "sk-key", frozenset({"anthropic", "openai"}))
         assert env["ANTHROPIC_BASE_URL"] == "http://localhost:4000"
         assert env["OPENAI_BASE_URL"] == "http://localhost:4000/v1"
         assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-key"
@@ -148,9 +143,7 @@ class TestBuildAgentEnv:
         assert env["ENABLE_TOOL_SEARCH"] == "true"
 
     def test_litellm_profile_exports_only_the_proxy_key(self):
-        env = build_agent_env(
-            {}, "http://localhost:4000/", "sk-key", frozenset({"litellm"})
-        )
+        env = build_agent_env({}, "http://localhost:4000/", "sk-key", frozenset({"litellm"}))
         assert env["LITELLM_PROXY_API_KEY"] == "sk-key"
         assert "ANTHROPIC_BASE_URL" not in env
         assert "OPENAI_BASE_URL" not in env
@@ -158,9 +151,7 @@ class TestBuildAgentEnv:
 
     def test_preserves_unrelated_env_and_does_not_mutate_input(self):
         base = {"PATH": "/usr/bin", "ANTHROPIC_API_KEY": "real-key"}
-        env = build_agent_env(
-            base, "http://localhost:4000", "sk-key", frozenset({"anthropic"})
-        )
+        env = build_agent_env(base, "http://localhost:4000", "sk-key", frozenset({"anthropic"}))
         assert env["PATH"] == "/usr/bin"
         assert base == {"PATH": "/usr/bin", "ANTHROPIC_API_KEY": "real-key"}
 
@@ -320,9 +311,7 @@ class TestOpencodeModelSync:
         assert "refused" in result.reason
 
     def test_non_200_is_reported(self):
-        result = opencode_model_sync_env(
-            {}, "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(500)
-        )
+        result = opencode_model_sync_env({}, "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(500))
         assert isinstance(result, ModelSyncSkipped)
         assert "HTTP 500" in result.reason
 
@@ -454,13 +443,37 @@ class TestCodexModelSync:
         slugs = [m["slug"] for m in json.loads((tmp_path / "litellm-models.json").read_text())["models"]]
         assert slugs == ["new"]
 
-    def test_defaults_to_dot_codex_in_home(self, tmp_path, monkeypatch):
-        monkeypatch.setattr("pathlib.Path.home", classmethod(lambda cls: tmp_path))
+    def test_catalog_is_replaced_whole_and_leaves_no_temp_files(self, tmp_path):
+        self._sync(self._listing(*(self._row(f"m{i}") for i in range(50))), tmp_path)
+        self._sync(self._listing(self._row("new")), tmp_path)
+        assert [p.name for p in tmp_path.iterdir()] == ["litellm-models.json"]
+        assert json.loads((tmp_path / "litellm-models.json").read_text())["models"][0]["slug"] == "new"
+
+    def test_defaults_to_dot_codex_in_home(self, tmp_path):
         result = codex_model_sync_args(
-            {}, "http://localhost:4000", "sk-key", get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m")))
+            {},
+            "http://localhost:4000",
+            "sk-key",
+            get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m"))),
+            home=lambda: tmp_path,
         )
         assert self._catalog_path(result) == str(tmp_path / ".codex" / "litellm-models.json")
 
+    def test_default_home_is_the_users(self):
+        assert _default_of(codex_model_sync_args, "home") == Path.home
+
+    def test_missing_base_instructions_is_reported_not_raised(self, tmp_path):
+        result = codex_model_sync_args(
+            {"CODEX_HOME": str(tmp_path)},
+            "http://localhost:4000",
+            "sk-key",
+            get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m"))),
+            instructions_path=tmp_path / "missing.md",
+        )
+        assert isinstance(result, ModelSyncSkipped)
+        assert "could not read" in result.reason
+        assert not (tmp_path / "litellm-models.json").exists()
+
     def test_unwritable_catalog_path_is_reported_not_raised(self, tmp_path):
         blocker = tmp_path / "file"
         blocker.write_text("")
@@ -528,7 +541,9 @@ class TestRunAgent:
         args = calls["args"]
         assert args[-2:] == ("exec", "hi")
         assert args[args.index('model_catalog_json="/tmp/c.json"') - 1] == "-c"
-        assert args.index('model_provider="litellm"') < args.index('model_catalog_json="/tmp/c.json"') < args.index("exec")
+        assert (
+            args.index('model_provider="litellm"') < args.index('model_catalog_json="/tmp/c.json"') < args.index("exec")
+        )
         assert calls["env"]["OPENAI_API_KEY"] == "sk-key"
         assert "model_catalog_json" not in json.dumps(calls["env"])
 
@@ -1214,10 +1229,7 @@ class TestAgentCommands:
         assert captured["api_key"] == "sk-key"
         assert captured["command"] == ["claude", "--resume", "-p", "hi"]
         assert captured["skip_verify"] is False
-        assert (
-            "routing Claude Code through proxy at http://localhost:4000"
-            in result.output
-        )
+        assert "routing Claude Code through proxy at http://localhost:4000" in result.output
 
     def test_codex_shows_friendly_name(self):
         captured = {}
@@ -1290,14 +1302,10 @@ class TestAgentCommands:
         with (
             patch(f"{AGENTS_MODULE}._is_interactive", return_value=True),
             patch(f"{AGENTS_MODULE}.login", fake_login),
-            patch(
-                f"{AGENTS_MODULE}.get_stored_api_key", return_value="sk-after-login"
-            ) as mock_get,
+            patch(f"{AGENTS_MODULE}.get_stored_api_key", return_value="sk-after-login") as mock_get,
             patch(
                 f"{AGENTS_MODULE}.run_agent",
-                side_effect=lambda base_url, api_key, command, **k: captured.update(
-                    api_key=api_key
-                ),
+                side_effect=lambda base_url, api_key, command, **k: captured.update(api_key=api_key),
             ),
         ):
             result = self.runner.invoke(

From 96bf276ab9ef475b3eb4384a803258d80804a3ed Mon Sep 17 00:00:00 2001
From: jesus <jesus@berri.ai>
Date: Wed, 9 Sep 2026 22:36:09 +0000
Subject: [PATCH 05/67] fix(cli): remove the temp catalog when the atomic
 replace fails

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/client/cli/commands/agents.py        | 6 +++++-
 tests/test_litellm/proxy/client/cli/test_agents.py | 7 +++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py
index 8e3698985af..0354aefa95c 100644
--- a/litellm/proxy/client/cli/commands/agents.py
+++ b/litellm/proxy/client/cli/commands/agents.py
@@ -453,7 +453,11 @@ def _replace_file(path: Path, text: str) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
         _ = tmp.write(text)
-    os.replace(tmp.name, path)
+    try:
+        os.replace(tmp.name, path)
+    except OSError:
+        Path(tmp.name).unlink(missing_ok=True)
+        raise
 
 
 def codex_model_sync_args(
diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py
index 75e42c3eaa0..3020b770e62 100644
--- a/tests/test_litellm/proxy/client/cli/test_agents.py
+++ b/tests/test_litellm/proxy/client/cli/test_agents.py
@@ -481,6 +481,13 @@ class TestCodexModelSync:
         assert isinstance(result, ModelSyncSkipped)
         assert "could not write" in result.reason
 
+    def test_failed_replace_is_reported_and_leaves_no_temp_file(self, tmp_path):
+        (tmp_path / "litellm-models.json").mkdir()
+        _, result = self._sync(self._listing(self._row("m")), tmp_path)
+        assert isinstance(result, ModelSyncSkipped)
+        assert "could not write" in result.reason
+        assert [p.name for p in tmp_path.iterdir()] == ["litellm-models.json"]
+
     def test_unreachable_proxy_is_reported_not_raised(self, tmp_path):
         def boom(*a, **k):
             raise requests.ConnectionError("refused")

From d5c7e279d7a1ca9f7e0d438c7e380b8b848a15fa Mon Sep 17 00:00:00 2001
From: shivam <shivam@berri.ai>
Date: Sat, 12 Sep 2026 18:05:24 +0000
Subject: [PATCH 06/67] fix(bedrock): sanitize client tool_call ids to Bedrock
 toolUseId constraints

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../prompt_templates/factory.py               | 27 ++++++-
 ...llm_core_utils_prompt_templates_factory.py | 73 +++++++++++++++++++
 2 files changed, 97 insertions(+), 3 deletions(-)

diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py
index ece619e3883..c3591e62a20 100644
--- a/litellm/litellm_core_utils/prompt_templates/factory.py
+++ b/litellm/litellm_core_utils/prompt_templates/factory.py
@@ -1515,6 +1515,23 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
     return sanitized
 
 
+_BEDROCK_TOOL_USE_ID_MAX_LEN: Final = 64
+_BEDROCK_TOOL_USE_ID_HASH_LEN: Final = 8
+
+
+def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str:
+    """
+    Bedrock Converse requires toolUseId to match [a-zA-Z0-9_.:-]+ and be at most 64 chars.
+    Over-long ids are truncated and suffixed with a short hash of the original so two ids
+    that only differ past the cut still map to distinct values.
+    """
+    sanitized: Final = re.sub(r"[^a-zA-Z0-9_.:-]", "_", tool_use_id) or "tool_use_id"
+    if len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN:
+        return sanitized
+    digest: Final = hashlib.sha256(tool_use_id.encode()).hexdigest()[:_BEDROCK_TOOL_USE_ID_HASH_LEN]
+    return f"{sanitized[: _BEDROCK_TOOL_USE_ID_MAX_LEN - _BEDROCK_TOOL_USE_ID_HASH_LEN - 1]}_{digest}"
+
+
 _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES: Final = {"application/pdf", "text/plain"}
 
 
@@ -3661,7 +3678,9 @@ def _convert_to_bedrock_tool_call_invoke(
                         if parsed_objects:
                             # First object keeps the original tool id.
                             for obj_idx, obj in enumerate(parsed_objects):
-                                block_id = tool_id if obj_idx == 0 else f"{tool_id}_{obj_idx}"
+                                block_id = _sanitize_bedrock_tool_use_id(
+                                    tool_id if obj_idx == 0 else f"{tool_id}_{obj_idx}"
+                                )
                                 bedrock_tool = BedrockToolUseBlock(input=obj, name=name, toolUseId=block_id)
                                 _parts_list.append(BedrockContentBlock(toolUse=bedrock_tool))
                             # cache_control applies to the whole original
@@ -3678,7 +3697,9 @@ def _convert_to_bedrock_tool_call_invoke(
                         # Fallback: no objects extracted — use empty dict.
                         arguments_dict = {}
 
-                bedrock_tool = BedrockToolUseBlock(input=arguments_dict, name=name, toolUseId=tool_id)
+                bedrock_tool = BedrockToolUseBlock(
+                    input=arguments_dict, name=name, toolUseId=_sanitize_bedrock_tool_use_id(tool_id)
+                )
                 bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
                 _parts_list.append(bedrock_content_block)
 
@@ -3849,7 +3870,7 @@ def _convert_to_bedrock_tool_call_result(
     tool_result_content_blocks, used_search_results = _build_bedrock_tool_result_content_blocks(message)
 
     message.get("name", "")
-    id: Final = str(message.get("tool_call_id", str(uuid.uuid4())))
+    id: Final = _sanitize_bedrock_tool_use_id(str(message.get("tool_call_id", str(uuid.uuid4()))))
 
     tool_result: Final = BedrockToolResultBlock(content=tool_result_content_blocks, toolUseId=id)
     if used_search_results:
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
index 66d10fd1407..32445b8b6ec 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
@@ -2,6 +2,7 @@ import base64
 import json
 import logging
 import os
+import re
 from typing import Final
 from unittest.mock import MagicMock, patch
 
@@ -2208,6 +2209,78 @@ def test_bedrock_tool_call_invoke_empty_arguments():
     assert result[0]["toolUse"]["input"] == {}
 
 
+_BEDROCK_TOOL_USE_ID_RE = re.compile(r"^[a-zA-Z0-9_.:-]{1,64}$")
+
+
+@pytest.mark.parametrize(
+    "tool_call_id",
+    [
+        "call_" + "x" * 100,
+        "call|with|pipes",
+        "call_" + "y" * 60 + "|end",
+        "call:ok.dots-and_under",
+    ],
+)
+def test_bedrock_tool_use_id_is_sanitized_consistently_for_invoke_and_result(tool_call_id):
+    """
+    Regression test for https://github.com/BerriAI/litellm/issues/34239: client-minted
+    tool_call ids longer than 64 chars or with chars outside [a-zA-Z0-9_.:-] made Bedrock
+    return a 400. The invoke and result paths must produce the same valid toolUseId so the
+    toolUse/toolResult pair still correlates.
+    """
+    invoke = _convert_to_bedrock_tool_call_invoke(
+        [
+            {
+                "id": tool_call_id,
+                "type": "function",
+                "function": {"name": "get_weather", "arguments": '{"location": "Boston"}'},
+            }
+        ]
+    )
+    result = _convert_to_bedrock_tool_call_result(
+        {"tool_call_id": tool_call_id, "role": "tool", "name": "get_weather", "content": "sunny"}
+    )
+    tool_use_id = invoke[0]["toolUse"]["toolUseId"]
+    assert _BEDROCK_TOOL_USE_ID_RE.match(tool_use_id)
+    assert result["toolResult"]["toolUseId"] == tool_use_id
+
+
+def test_bedrock_tool_use_id_valid_ids_pass_through_unchanged():
+    result = _convert_to_bedrock_tool_call_result(
+        {"tool_call_id": "tooluse_Ab.c:1-2_3", "role": "tool", "name": "f", "content": "ok"}
+    )
+    assert result["toolResult"]["toolUseId"] == "tooluse_Ab.c:1-2_3"
+
+
+def test_bedrock_tool_use_id_truncation_keeps_distinct_ids_distinct():
+    prefix = "call_" + "z" * 70
+    ids = {
+        _convert_to_bedrock_tool_call_result(
+            {"tool_call_id": f"{prefix}{suffix}", "role": "tool", "name": "f", "content": "ok"}
+        )["toolResult"]["toolUseId"]
+        for suffix in ("a", "b")
+    }
+    assert len(ids) == 2
+    assert all(len(i) == 64 for i in ids)
+
+
+def test_bedrock_tool_call_invoke_concatenated_json_long_id_stays_within_limit():
+    long_id = "call_" + "q" * 62
+    result = _convert_to_bedrock_tool_call_invoke(
+        [
+            {
+                "id": long_id,
+                "type": "function",
+                "function": {"name": "run", "arguments": '{"cmd":"a"}{"cmd":"b"}'},
+            }
+        ]
+    )
+    ids = [block["toolUse"]["toolUseId"] for block in result]
+    assert len(ids) == 2
+    assert len(set(ids)) == 2
+    assert all(_BEDROCK_TOOL_USE_ID_RE.match(i) for i in ids)
+
+
 def test_bedrock_tool_call_invoke_concatenated_json():
     """
     Tool call whose arguments contain multiple concatenated JSON objects

From 646fd537407d614ecde33ef3f361024569f6b174 Mon Sep 17 00:00:00 2001
From: shivam <shivam@berri.ai>
Date: Sat, 12 Sep 2026 18:17:52 +0000
Subject: [PATCH 07/67] fix(bedrock): hash-suffix tool ids whose chars were
 rewritten so they cannot collide

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/litellm_core_utils/prompt_templates/factory.py |  6 +++---
 ...test_litellm_core_utils_prompt_templates_factory.py | 10 ++++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py
index c3591e62a20..f7f4a964c9b 100644
--- a/litellm/litellm_core_utils/prompt_templates/factory.py
+++ b/litellm/litellm_core_utils/prompt_templates/factory.py
@@ -1522,11 +1522,11 @@ _BEDROCK_TOOL_USE_ID_HASH_LEN: Final = 8
 def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str:
     """
     Bedrock Converse requires toolUseId to match [a-zA-Z0-9_.:-]+ and be at most 64 chars.
-    Over-long ids are truncated and suffixed with a short hash of the original so two ids
-    that only differ past the cut still map to distinct values.
+    Ids that need rewriting get a short hash of the original appended so two ids that only
+    differ in a replaced char or past the cut still map to distinct values.
     """
     sanitized: Final = re.sub(r"[^a-zA-Z0-9_.:-]", "_", tool_use_id) or "tool_use_id"
-    if len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN:
+    if sanitized == tool_use_id and len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN:
         return sanitized
     digest: Final = hashlib.sha256(tool_use_id.encode()).hexdigest()[:_BEDROCK_TOOL_USE_ID_HASH_LEN]
     return f"{sanitized[: _BEDROCK_TOOL_USE_ID_MAX_LEN - _BEDROCK_TOOL_USE_ID_HASH_LEN - 1]}_{digest}"
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
index 32445b8b6ec..fe8a9bd5205 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
@@ -2264,6 +2264,16 @@ def test_bedrock_tool_use_id_truncation_keeps_distinct_ids_distinct():
     assert all(len(i) == 64 for i in ids)
 
 
+def test_bedrock_tool_use_id_replaced_chars_do_not_collide_with_existing_ids():
+    ids = {
+        _convert_to_bedrock_tool_call_result({"tool_call_id": i, "role": "tool", "name": "f", "content": "ok"})[
+            "toolResult"
+        ]["toolUseId"]
+        for i in ("call|x", "call_x")
+    }
+    assert len(ids) == 2
+
+
 def test_bedrock_tool_call_invoke_concatenated_json_long_id_stays_within_limit():
     long_id = "call_" + "q" * 62
     result = _convert_to_bedrock_tool_call_invoke(

From 222f283c9352ca828a7db7641b552de43e017af0 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 12 Sep 2026 15:55:03 -0700
Subject: [PATCH 08/67] fix(cli): read the Codex catalog back before launch and
 cover every ModelInfo schema

Codex 0.130 and 0.145 require supports_reasoning_summaries and
supports_parallel_tool_calls on every catalog entry, so a catalog written
for 0.154 made those releases exit at startup with a parse error. Every
field some release since 0.105.0 deserializes without a default is now
written, with Codex's own fallback values, and the catalog is read back
once through the installed binary (`codex debug models`) before launch.
A Codex that rejects it, or one older than 0.130 with no such command,
gets the skip notice and launches on its built-in catalog instead.
---
 litellm/proxy/client/cli/README.md            |   2 +-
 litellm/proxy/client/cli/commands/agents.py   |  80 +++++++++--
 .../proxy/client/cli/test_agents.py           | 130 +++++++++++++++++-
 3 files changed, 190 insertions(+), 22 deletions(-)

diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md
index 3b0ff9d7add..046786d9557 100644
--- a/litellm/proxy/client/cli/README.md
+++ b/litellm/proxy/client/cli/README.md
@@ -490,7 +490,7 @@ lite codex exec "summarize the repo"
 
 Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
 
-The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway.
+The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; before launching, `lite codex` has the installed Codex read that file back (`codex debug models`), and when the fetch, the write or that read-back fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving the rejected file in place.
 
 pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.
 
diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py
index f424e07968e..8a15153ea28 100644
--- a/litellm/proxy/client/cli/commands/agents.py
+++ b/litellm/proxy/client/cli/commands/agents.py
@@ -69,6 +69,7 @@ CODEX_PROXY_PROVIDER: Final = "litellm"
 CODEX_HOME_ENV: Final = "CODEX_HOME"
 CODEX_MODEL_CATALOG_FILENAME: Final = "litellm-models.json"
 _CODEX_BASE_INSTRUCTIONS_PATH: Final = Path(__file__).with_name("codex_base_instructions.md")
+_CODEX_PREFLIGHT_TIMEOUT_SECONDS: Final = 10.0
 
 
 class AgentRunError(Exception):
@@ -399,9 +400,11 @@ class _CodexTruncationPolicy(BaseModel):
 class _CodexModel(BaseModel):
     """One `ModelInfo` entry of a Codex model catalog.
 
-    Every field Codex's deserializer has no default for is spelled out here; the
-    values match the fallback metadata Codex uses today for a model slug it
-    does not know, so picking a proxy model behaves the same as `codex -m` did.
+    Every field that some Codex release since `model_catalog_json` appeared
+    (0.105.0) deserializes without a default is spelled out here, so one catalog
+    parses on all of them; the values match the fallback metadata Codex uses for
+    a model slug it does not know, so picking a proxy model behaves the same as
+    `codex -m` did.
     """
 
     slug: str
@@ -415,6 +418,8 @@ class _CodexModel(BaseModel):
     availability_nux: None = None
     upgrade: None = None
     support_verbosity: Literal[False] = False
+    supports_reasoning_summaries: Literal[False] = False
+    supports_parallel_tool_calls: Literal[False] = False
     default_verbosity: None = None
     apply_patch_tool_type: None = None
     truncation_policy: _CodexTruncationPolicy = _CodexTruncationPolicy()
@@ -470,12 +475,48 @@ def _replace_file(path: Path, text: str) -> None:
         raise
 
 
+def _codex_catalog_rejection(
+    binary: str,
+    override: str,
+    env: Mapping[str, str],
+    *,
+    run: Callable[..., subprocess.CompletedProcess[str]],
+) -> str | None:
+    """Why the installed Codex refuses the catalog, or None once it reads the file back.
+
+    `codex debug models` parses the catalog the way a launch does, so a Codex
+    whose ModelInfo schema disagrees with the one written here fails now, with
+    the sync skipped, instead of exiting on startup. Releases before 0.130.0
+    have no `debug models` and fail the same way. A batch shim goes through
+    cmd.exe exactly as the launch will.
+    """
+    name: Final = os.path.basename(binary)
+    command: Final = _windows_command(binary, (binary, "-c", override, "debug", "models"))
+    try:
+        completed: Final = run(
+            command,
+            env=dict(env),
+            stdin=subprocess.DEVNULL,
+            capture_output=True,
+            text=True,
+            timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
+        )
+    except (OSError, subprocess.TimeoutExpired) as e:
+        return f"`{name} debug models` failed: {e}"
+    if completed.returncode == 0:
+        return None
+    lines: Final = completed.stderr.strip().splitlines()
+    return f"`{name} debug models` exited {completed.returncode}: {lines[0] if lines else 'no output'}"
+
+
 def codex_model_sync_args(
     base_env: Mapping[str, str],
     base_url: str,
     api_key: str,
     *,
+    binary: str = "codex",
     get: Callable[..., requests.Response] = requests.get,
+    run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
     home: Callable[[], Path] = Path.home,
     instructions_path: Path = _CODEX_BASE_INSTRUCTIONS_PATH,
 ) -> ModelSyncArgs | ModelSyncSkipped:
@@ -483,9 +524,11 @@ def codex_model_sync_args(
 
     Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
     must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
-    atomically replaced on every launch. The key never lands in the file. A
-    failed fetch, read or write is reported rather than raised: Codex still
-    launches with its built-in catalog and takes a proxy model by name via -m.
+    atomically replaced on every launch, then read back once through the Codex
+    at `binary` before it is handed over. The key never lands in the file. A
+    failed fetch, read, write or read-back is reported rather than raised: Codex
+    still launches with its built-in catalog and takes a proxy model by name via
+    -m, and a rejected file stays on disk to be looked at.
     """
     listing: Final = _fetch_model_listing(base_url, api_key, get=get)
     if isinstance(listing, ModelSyncSkipped):
@@ -502,32 +545,39 @@ def codex_model_sync_args(
         _replace_file(path, catalog)
     except OSError as e:
         return ModelSyncSkipped(f"could not write {path}: {e}")
-    return ModelSyncArgs(("-c", f"model_catalog_json={json.dumps(str(path))}"))
+    override: Final = f"model_catalog_json={json.dumps(str(path))}"
+    rejection: Final = _codex_catalog_rejection(binary, override, base_env, run=run)
+    if rejection is not None:
+        return ModelSyncSkipped(rejection)
+    return ModelSyncArgs(("-c", override))
 
 
 def agent_model_sync_env(
-    command: str,
+    binary: str,
     base_env: Mapping[str, str],
     base_url: str,
     api_key: str,
     skip_verify: bool,
     *,
     get: Callable[..., requests.Response] = requests.get,
+    run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
 ) -> ModelSyncResult:
     """Extra env or args an agent needs to see the proxy's model list.
 
-    OpenCode takes it as env, Codex as a `-c` override; Claude Code discovers
-    models itself through CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY.
-    skip_verify means the caller wants no pre-launch proxy call at all, so the
-    listing is skipped too rather than hanging on an offline proxy.
+    binary is the resolved path the launch will run (`codex.cmd` on a Windows
+    npm install). OpenCode takes the list as env, Codex as a `-c` override that
+    binary has read back first; Claude Code discovers models itself through
+    CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY. skip_verify means the caller
+    wants no pre-launch proxy call at all, so the listing is skipped too rather
+    than hanging on an offline proxy.
     """
-    agent: Final = os.path.basename(command)
+    agent: Final = os.path.splitext(os.path.basename(binary))[0]
     if agent not in ("opencode", "codex"):
         return _NO_EXTRA_ENV
     if skip_verify:
         return ModelSyncSkipped(f"{_SKIP_VERIFY_FLAG} was passed")
     if agent == "codex":
-        return codex_model_sync_args(base_env, base_url, api_key, get=get)
+        return codex_model_sync_args(base_env, base_url, api_key, binary=binary, get=get, run=run)
     return opencode_model_sync_env(base_env, base_url, api_key, get=get)
 
 
@@ -682,7 +732,7 @@ def run_agent(
         verify(base_url, api_key)
 
     env_before_sync: Final = base_env if base_env is not None else os.environ
-    synced: Final = sync_models(command[0], env_before_sync, base_url, api_key, skip_verify)
+    synced: Final = sync_models(binary, env_before_sync, base_url, api_key, skip_verify)
     if isinstance(synced, ModelSyncSkipped):
         warn(f"litellm: not syncing {display_name} models from the proxy: {synced.reason}")
 
diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py
index b122ea7b20e..8e76e4745f1 100644
--- a/tests/test_litellm/proxy/client/cli/test_agents.py
+++ b/tests/test_litellm/proxy/client/cli/test_agents.py
@@ -1,6 +1,7 @@
 import inspect
 import json
 import os
+import subprocess
 import sys
 from pathlib import Path
 from unittest.mock import patch
@@ -56,6 +57,17 @@ class _Recorder:
         return self.returns
 
 
+class _FakeRun:
+    def __init__(self, returncode=0, stderr=""):
+        self.returncode = returncode
+        self.stderr = stderr
+        self.calls = []
+
+    def __call__(self, args, **kwargs):
+        self.calls.append((args, kwargs))
+        return subprocess.CompletedProcess(args, self.returncode, "", self.stderr)
+
+
 class _FakeJsonResponse:
     def __init__(self, status_code, payload=None):
         self.status_code = status_code
@@ -365,7 +377,7 @@ class TestCodexModelSync:
     def _row(model_id, **extra):
         return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra}
 
-    def _sync(self, listing, codex_home, base_url="http://localhost:4000/"):
+    def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None):
         captured = {}
 
         def fake_get(url, headers, timeout):
@@ -373,7 +385,13 @@ class TestCodexModelSync:
             captured["headers"] = headers
             return _FakeResponse(200, listing)
 
-        result = codex_model_sync_args({"CODEX_HOME": str(codex_home)}, base_url, "sk-key", get=fake_get)
+        result = codex_model_sync_args(
+            {"CODEX_HOME": str(codex_home)},
+            base_url,
+            "sk-key",
+            get=fake_get,
+            run=_FakeRun() if run is None else run,
+        )
         return captured, result
 
     @staticmethod
@@ -411,6 +429,8 @@ class TestCodexModelSync:
         assert entry["truncation_policy"] == {"mode": "bytes", "limit": 10000}
         assert entry["experimental_supported_tools"] == []
         assert entry["support_verbosity"] is False
+        assert entry["supports_reasoning_summaries"] is False
+        assert entry["supports_parallel_tool_calls"] is False
         for nullable in ("description", "availability_nux", "upgrade", "default_verbosity", "apply_patch_tool_type"):
             assert nullable in entry and entry[nullable] is None
         assert entry["base_instructions"].startswith("You are a coding agent running in the Codex CLI")
@@ -457,6 +477,7 @@ class TestCodexModelSync:
             "http://localhost:4000",
             "sk-key",
             get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m"))),
+            run=_FakeRun(),
             home=lambda: tmp_path,
         )
         assert self._catalog_path(result) == str(tmp_path / ".codex" / "litellm-models.json")
@@ -510,17 +531,108 @@ class TestCodexModelSync:
         assert isinstance(result, ModelSyncSkipped)
         assert reason in result.reason
 
-    @pytest.mark.parametrize("command", ["codex", "/opt/bin/codex"])
-    def test_codex_syncs_through_the_agent_dispatch(self, tmp_path, command):
+    @pytest.mark.parametrize("binary", ["codex", "/opt/bin/codex", "codex.cmd", "/c/npm/codex.CMD"])
+    def test_codex_syncs_through_the_agent_dispatch_with_the_binary_it_will_run(self, tmp_path, binary):
+        run = _FakeRun()
         result = agent_model_sync_env(
-            command,
+            binary,
             {"CODEX_HOME": str(tmp_path)},
             "http://localhost:4000",
             "sk-key",
             False,
             get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m"))),
+            run=run,
         )
         assert self._catalog_path(result) == str(tmp_path / "litellm-models.json")
+        assert binary in run.calls[0][0]
+
+    def test_opencode_dispatch_never_runs_codex(self):
+        def boom(*a, **k):
+            raise AssertionError("only the Codex sync reads its catalog back")
+
+        result = agent_model_sync_env(
+            "opencode",
+            {},
+            "http://localhost:4000",
+            "sk-key",
+            False,
+            get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m"))),
+            run=boom,
+        )
+        assert "OPENCODE_CONFIG_CONTENT" in result
+
+    def test_catalog_is_read_back_through_codex_before_launch(self, tmp_path):
+        run = _FakeRun()
+        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=run)
+        path = self._catalog_path(result)
+
+        assert len(run.calls) == 1
+        command, options = run.calls[0]
+        assert command == ("codex", "-c", f"model_catalog_json={json.dumps(path)}", "debug", "models")
+        assert options["env"] == {"CODEX_HOME": str(tmp_path)}
+        assert options["stdin"] is subprocess.DEVNULL
+        assert options["capture_output"] is True
+        assert options["text"] is True
+        assert options["timeout"] == 10
+
+    def test_codex_rejecting_the_catalog_skips_the_sync_and_keeps_the_file(self, tmp_path):
+        stderr = (
+            "Error: failed to parse model_catalog_json path `/home/me/.codex/litellm-models.json` as JSON: "
+            "missing field `supports_parallel_tool_calls` at line 1 column 21648\n"
+        )
+        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=_FakeRun(1, stderr))
+        assert isinstance(result, ModelSyncSkipped)
+        assert result.reason == (
+            "`codex debug models` exited 1: Error: failed to parse model_catalog_json path "
+            "`/home/me/.codex/litellm-models.json` as JSON: missing field `supports_parallel_tool_calls` "
+            "at line 1 column 21648"
+        )
+        assert (tmp_path / "litellm-models.json").exists()
+
+    def test_codex_without_debug_models_skips_the_sync(self, tmp_path):
+        stderr = "error: unrecognized subcommand 'models'\n\nUsage: codex debug [OPTIONS] <COMMAND>\n"
+        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=_FakeRun(2, stderr))
+        assert isinstance(result, ModelSyncSkipped)
+        assert result.reason == "`codex debug models` exited 2: error: unrecognized subcommand 'models'"
+
+    def test_codex_failing_silently_is_reported(self, tmp_path):
+        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=_FakeRun(1))
+        assert isinstance(result, ModelSyncSkipped)
+        assert result.reason == "`codex debug models` exited 1: no output"
+
+    @pytest.mark.parametrize(
+        "error", [OSError("codex vanished"), subprocess.TimeoutExpired("codex", 10)], ids=["oserror", "timeout"]
+    )
+    def test_unrunnable_preflight_is_reported_not_raised(self, tmp_path, error):
+        def failing_run(*a, **k):
+            raise error
+
+        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=failing_run)
+        assert isinstance(result, ModelSyncSkipped)
+        assert result.reason.startswith("`codex debug models` failed: ")
+        assert str(error) in result.reason
+
+    def test_windows_shim_preflight_goes_through_cmd_exe(self, tmp_path):
+        shim = _WINDOWS_CLAUDE_CMD.replace("claude", "codex")
+        run = _FakeRun()
+        result = codex_model_sync_args(
+            {"CODEX_HOME": str(tmp_path)},
+            "http://localhost:4000",
+            "sk-key",
+            binary=shim,
+            get=lambda *a, **k: _FakeResponse(200, self._listing(self._row("m"))),
+            run=run,
+        )
+        override = f"model_catalog_json={json.dumps(self._catalog_path(result))}"
+        doubled = override.replace('"', '""')
+        assert run.calls[0][0] == f'{_CMD_PREFIX}""{shim}" "-c" "{doubled}" "debug" "models""'
+
+    def test_default_binary_is_codex_on_path(self):
+        assert _default_of(codex_model_sync_args, "binary") == "codex"
+
+    def test_default_runner_is_subprocess_run(self):
+        assert _default_of(codex_model_sync_args, "run") is subprocess.run
+        assert _default_of(agent_model_sync_env, "run") is subprocess.run
 
     def test_skip_verify_keeps_the_launch_offline(self):
         def boom(*a, **k):
@@ -593,7 +705,13 @@ class TestRunAgent:
             launcher=lambda p, a, e: order.append("launch"),
         )
         assert order == ["verify", "sync", "launch"]
-        assert calls["args"] == ("opencode", {"HOME": "/home/me"}, "http://localhost:4000", "sk-key", False)
+        assert calls["args"] == (
+            "/usr/local/bin/opencode",
+            {"HOME": "/home/me"},
+            "http://localhost:4000",
+            "sk-key",
+            False,
+        )
 
     def test_unreachable_proxy_is_not_asked_for_models(self):
         def failing_verify(*a):

From 5b9153f5ea26897c3b138206523f75974701f8e8 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 12 Sep 2026 16:42:52 -0700
Subject: [PATCH 09/67] feat(cli): keep the installed Codex's entries for proxy
 models it already knows

`lite codex` now asks the installed Codex for its own model list through
`codex debug models` before writing the catalog. A proxy model whose id
matches a stock Codex slug keeps that Codex's entry (reasoning levels,
base instructions, context window and the rest) and only its picker
position, visibility and upgrade nudge come from the proxy. Unknown slugs
still get the plain entry built from the bundled base instructions. The
catalog directory is created before the stock call so a fresh CODEX_HOME
does not make Codex refuse to run
---
 litellm/proxy/client/cli/README.md            |   2 +-
 litellm/proxy/client/cli/commands/agents.py   | 148 +++++++++++----
 .../proxy/client/cli/test_agents.py           | 173 ++++++++++++++++--
 3 files changed, 263 insertions(+), 60 deletions(-)

diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md
index 046786d9557..965c90d0430 100644
--- a/litellm/proxy/client/cli/README.md
+++ b/litellm/proxy/client/cli/README.md
@@ -490,7 +490,7 @@ lite codex exec "summarize the repo"
 
 Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
 
-The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; before launching, `lite codex` has the installed Codex read that file back (`codex debug models`), and when the fetch, the write or that read-back fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving the rejected file in place.
+The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-<UTF-8 hex of the group name>` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=<path>` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place.
 
 pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/<id>`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. Your own flags come after the injected pin, so `lite pi --model litellm/<other-id>` wins, and inside the TUI the `/model` picker lists every synced litellm model.
 
diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py
index 8a15153ea28..1422514372a 100644
--- a/litellm/proxy/client/cli/commands/agents.py
+++ b/litellm/proxy/client/cli/commands/agents.py
@@ -13,7 +13,7 @@ from typing import Final, Literal, TypeAlias
 
 import click
 import requests
-from pydantic import BaseModel, TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from .auth import CliContextObj, context_secret_vault, get_stored_api_key, login
 from .claude_settings import ClaudeSettingsError, install_statusline_script
@@ -398,13 +398,13 @@ class _CodexTruncationPolicy(BaseModel):
 
 
 class _CodexModel(BaseModel):
-    """One `ModelInfo` entry of a Codex model catalog.
+    """One `ModelInfo` entry of a Codex model catalog for a model the installed Codex does not know.
 
     Every field that some Codex release since `model_catalog_json` appeared
     (0.105.0) deserializes without a default is spelled out here, so one catalog
     parses on all of them; the values match the fallback metadata Codex uses for
-    a model slug it does not know, so picking a proxy model behaves the same as
-    `codex -m` did.
+    a model slug it does not know, so picking such a proxy model behaves the
+    same as `codex -m` did.
     """
 
     slug: str
@@ -428,31 +428,77 @@ class _CodexModel(BaseModel):
     base_instructions: str
 
 
+class _StockCodexUpgrade(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    model: str
+
+
+class _StockCodexModel(BaseModel):
+    """One `ModelInfo` entry as the installed Codex prints it from `codex debug models`.
+
+    Only the fields the sync rewrites are named; everything else that release
+    knows about the model (its reasoning levels, prompt, tool support) rides
+    along untouched, whatever the release's schema.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    slug: str
+    priority: int
+    visibility: str
+    upgrade: _StockCodexUpgrade | None = None
+
+
+class _StockCodexCatalog(BaseModel):
+    models: tuple[_StockCodexModel, ...]
+
+
 class _CodexCatalog(BaseModel):
-    models: tuple[_CodexModel, ...]
+    models: tuple[_CodexModel | _StockCodexModel, ...]
 
 
-def codex_model_catalog(models: Sequence[ListedModel], instructions: str) -> str | None:
+def _codex_catalog_entry(
+    priority: int,
+    listed: ListedModel,
+    stock: _StockCodexModel | None,
+    served: frozenset[str],
+    instructions: str,
+) -> _CodexModel | _StockCodexModel:
+    if stock is None:
+        return _CodexModel(
+            slug=listed.id,
+            display_name=listed.id,
+            priority=priority,
+            context_window=listed.max_input_tokens,
+            base_instructions=instructions,
+        )
+    upgrade: Final = stock.upgrade if stock.upgrade is not None and stock.upgrade.model in served else None
+    return stock.model_copy(update={"priority": priority, "visibility": "list", "upgrade": upgrade})
+
+
+def codex_model_catalog(
+    models: Sequence[ListedModel], stock: Sequence[_StockCodexModel], instructions: str
+) -> str | None:
     """The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
 
     Codex refuses an empty catalog, hence None instead of `{"models": []}`.
-    Passing a catalog replaces Codex's built-in one, so every entry carries the
-    same base instructions Codex itself uses, otherwise the agent would run
-    without a system prompt.
+    Passing a catalog replaces Codex's built-in one, so a proxy model the
+    installed Codex knows keeps that Codex's own entry and the proxy only
+    decides its place in the picker: the listing orders it, lists it even when
+    Codex hides it, and keeps Codex's upgrade nudge only when the model it
+    points at is served too. A model Codex does not know gets the fallback
+    entry, with the same base instructions Codex itself uses so the agent never
+    runs without a system prompt.
     """
     chat_models: Final = _chat_models(models)
     if not chat_models:
         return None
+    served: Final = frozenset(m.id for m in chat_models)
+    known: Final = MappingProxyType({m.slug: m for m in stock})
     catalog: Final = _CodexCatalog(
         models=tuple(
-            _CodexModel(
-                slug=m.id,
-                display_name=m.id,
-                priority=index,
-                context_window=m.max_input_tokens,
-                base_instructions=instructions,
-            )
-            for index, m in enumerate(chat_models)
+            _codex_catalog_entry(index, m, known.get(m.id), served, instructions) for index, m in enumerate(chat_models)
         )
     )
     return catalog.model_dump_json()
@@ -465,7 +511,6 @@ def codex_model_catalog_path(env: Mapping[str, str], *, home: Callable[[], Path]
 
 
 def _replace_file(path: Path, text: str) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
     with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, delete=False) as tmp:
         _ = tmp.write(text)
     try:
@@ -475,23 +520,23 @@ def _replace_file(path: Path, text: str) -> None:
         raise
 
 
-def _codex_catalog_rejection(
+def _codex_debug_models(
     binary: str,
-    override: str,
+    args: Sequence[str],
     env: Mapping[str, str],
     *,
     run: Callable[..., subprocess.CompletedProcess[str]],
-) -> str | None:
-    """Why the installed Codex refuses the catalog, or None once it reads the file back.
+) -> str | ModelSyncSkipped:
+    """What `codex debug models` prints with `args` in front, or why the installed Codex could not run it.
 
-    `codex debug models` parses the catalog the way a launch does, so a Codex
-    whose ModelInfo schema disagrees with the one written here fails now, with
-    the sync skipped, instead of exiting on startup. Releases before 0.130.0
-    have no `debug models` and fail the same way. A batch shim goes through
+    The command prints the catalog Codex would launch with, without touching
+    the network, so it lists the installed Codex's own models and parses a
+    catalog override the way a launch does. Releases before 0.130.0 have no
+    such command and are reported the same way. A batch shim goes through
     cmd.exe exactly as the launch will.
     """
     name: Final = os.path.basename(binary)
-    command: Final = _windows_command(binary, (binary, "-c", override, "debug", "models"))
+    command: Final = _windows_command(binary, (binary, *args, "debug", "models"))
     try:
         completed: Final = run(
             command,
@@ -502,11 +547,25 @@ def _codex_catalog_rejection(
             timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
         )
     except (OSError, subprocess.TimeoutExpired) as e:
-        return f"`{name} debug models` failed: {e}"
+        return ModelSyncSkipped(f"`{name} debug models` failed: {e}")
     if completed.returncode == 0:
-        return None
+        return completed.stdout
     lines: Final = completed.stderr.strip().splitlines()
-    return f"`{name} debug models` exited {completed.returncode}: {lines[0] if lines else 'no output'}"
+    detail: Final = lines[0] if lines else "no output"
+    return ModelSyncSkipped(f"`{name} debug models` exited {completed.returncode}: {detail}")
+
+
+def _stock_codex_models(
+    binary: str, env: Mapping[str, str], *, run: Callable[..., subprocess.CompletedProcess[str]]
+) -> tuple[_StockCodexModel, ...] | ModelSyncSkipped:
+    printed: Final = _codex_debug_models(binary, (), env, run=run)
+    if isinstance(printed, ModelSyncSkipped):
+        return printed
+    try:
+        return _StockCodexCatalog.model_validate_json(printed).models
+    except ValidationError as e:
+        name: Final = os.path.basename(binary)
+        return ModelSyncSkipped(f"`{name} debug models` printed no model catalog: {e.errors()[0]['msg']}")
 
 
 def codex_model_sync_args(
@@ -524,11 +583,13 @@ def codex_model_sync_args(
 
     Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
     must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
-    atomically replaced on every launch, then read back once through the Codex
-    at `binary` before it is handed over. The key never lands in the file. A
-    failed fetch, read, write or read-back is reported rather than raised: Codex
-    still launches with its built-in catalog and takes a proxy model by name via
-    -m, and a rejected file stays on disk to be looked at.
+    atomically replaced on every launch. The Codex at `binary` first lists its
+    own models, so the ones the proxy serves keep that Codex's entries, and then
+    reads the file back once before it is handed over. The key never lands in
+    the file. A failed fetch, read, listing, write or read-back is reported
+    rather than raised: Codex still launches with its built-in catalog and takes
+    a proxy model by name via -m, and a rejected file stays on disk to be looked
+    at.
     """
     listing: Final = _fetch_model_listing(base_url, api_key, get=get)
     if isinstance(listing, ModelSyncSkipped):
@@ -537,18 +598,25 @@ def codex_model_sync_args(
         instructions: Final = instructions_path.read_text(encoding="utf-8")
     except OSError as e:
         return ModelSyncSkipped(f"could not read {instructions_path}: {e}")
-    catalog: Final = codex_model_catalog(listing, instructions)
+    path: Final = codex_model_catalog_path(base_env, home=home)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+    except OSError as e:
+        return ModelSyncSkipped(f"could not write {path}: {e}")
+    stock: Final = _stock_codex_models(binary, base_env, run=run)
+    if isinstance(stock, ModelSyncSkipped):
+        return stock
+    catalog: Final = codex_model_catalog(listing, stock, instructions)
     if catalog is None:
         return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
-    path: Final = codex_model_catalog_path(base_env, home=home)
     try:
         _replace_file(path, catalog)
     except OSError as e:
         return ModelSyncSkipped(f"could not write {path}: {e}")
     override: Final = f"model_catalog_json={json.dumps(str(path))}"
-    rejection: Final = _codex_catalog_rejection(binary, override, base_env, run=run)
-    if rejection is not None:
-        return ModelSyncSkipped(rejection)
+    read_back: Final = _codex_debug_models(binary, ("-c", override), base_env, run=run)
+    if isinstance(read_back, ModelSyncSkipped):
+        return read_back
     return ModelSyncArgs(("-c", override))
 
 
diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py
index 8e76e4745f1..cc7c3a14f44 100644
--- a/tests/test_litellm/proxy/client/cli/test_agents.py
+++ b/tests/test_litellm/proxy/client/cli/test_agents.py
@@ -57,14 +57,109 @@ class _Recorder:
         return self.returns
 
 
+_STOCK_REASONING_LEVELS = [
+    {"effort": "low", "description": "Fast responses with lighter reasoning"},
+    {"effort": "medium", "description": "Balances speed and reasoning depth for everyday tasks"},
+    {"effort": "high", "description": "Greater reasoning depth for complex problems"},
+]
+
+_STOCK_MODELS = {
+    "gpt-5.6-terra": {
+        "slug": "gpt-5.6-terra",
+        "display_name": "GPT-5.6 Terra",
+        "description": "Balanced agentic coding model for everyday work.",
+        "default_reasoning_level": "medium",
+        "supported_reasoning_levels": _STOCK_REASONING_LEVELS,
+        "shell_type": "unified_exec",
+        "visibility": "list",
+        "supported_in_api": True,
+        "priority": 7,
+        "availability_nux": None,
+        "upgrade": None,
+        "base_instructions": "You are Codex, a coding agent based on GPT-5.6.",
+        "apply_patch_tool_type": "freeform",
+        "supports_parallel_tool_calls": True,
+        "context_window": 272000,
+        "comp_hash": "terra-hash",
+    },
+    "gpt-5.5": {
+        "slug": "gpt-5.5",
+        "display_name": "GPT-5.5",
+        "description": "Frontier model for complex coding, research, and real-world work.",
+        "default_reasoning_level": "medium",
+        "supported_reasoning_levels": _STOCK_REASONING_LEVELS,
+        "shell_type": "unified_exec",
+        "visibility": "list",
+        "supported_in_api": True,
+        "priority": 12,
+        "availability_nux": None,
+        "upgrade": None,
+        "base_instructions": "You are Codex, a coding agent based on GPT-5.",
+        "apply_patch_tool_type": "freeform",
+        "supports_parallel_tool_calls": True,
+        "context_window": 272000,
+        "comp_hash": "gpt-5.5-hash",
+    },
+    "gpt-5.4": {
+        "slug": "gpt-5.4",
+        "display_name": "GPT-5.4",
+        "description": "Strong model for everyday coding.",
+        "default_reasoning_level": "medium",
+        "supported_reasoning_levels": _STOCK_REASONING_LEVELS,
+        "shell_type": "unified_exec",
+        "visibility": "hide",
+        "supported_in_api": True,
+        "priority": 16,
+        "availability_nux": None,
+        "upgrade": {
+            "model": "gpt-5.6-terra",
+            "migration_markdown": "GPT-5.4 is no longer available. Switch to GPT-5.6 Terra to continue.",
+            "retirement_at": "2026-08-31T19:00:00Z",
+        },
+        "base_instructions": "You are Codex, a coding agent based on GPT-5.",
+        "apply_patch_tool_type": "freeform",
+        "supports_parallel_tool_calls": True,
+        "context_window": 272000,
+        "comp_hash": "gpt-5.4-hash",
+    },
+    "codex-auto-review": {
+        "slug": "codex-auto-review",
+        "display_name": "Codex Auto Review",
+        "description": None,
+        "supported_reasoning_levels": [],
+        "shell_type": "unified_exec",
+        "visibility": "hide",
+        "supported_in_api": False,
+        "priority": 43,
+        "availability_nux": None,
+        "upgrade": None,
+        "base_instructions": "You are Codex, reviewing a change.",
+        "apply_patch_tool_type": None,
+        "supports_parallel_tool_calls": True,
+        "context_window": 272000,
+        "comp_hash": "review-hash",
+    },
+}
+
+_STOCK_CATALOG = json.dumps({"models": list(_STOCK_MODELS.values())})
+
+
 class _FakeRun:
-    def __init__(self, returncode=0, stderr=""):
+    """A `codex` that prints `stock` from a bare `debug models` and answers a catalog override with `returncode`.
+
+    `stock=None` is a Codex with no `debug models` at all: every call answers with `returncode` and `stderr`.
+    """
+
+    def __init__(self, returncode=0, stderr="", stock=_STOCK_CATALOG):
         self.returncode = returncode
         self.stderr = stderr
+        self.stock = stock
         self.calls = []
 
     def __call__(self, args, **kwargs):
         self.calls.append((args, kwargs))
+        if self.stock is not None and "model_catalog_json=" not in str(args):
+            return subprocess.CompletedProcess(args, 0, self.stock, "")
         return subprocess.CompletedProcess(args, self.returncode, "", self.stderr)
 
 
@@ -415,10 +510,36 @@ class TestCodexModelSync:
         assert "sk-key" not in text
         catalog = json.loads(text)
         assert [m["slug"] for m in catalog["models"]] == ["gpt-5.5", "claude-opus-4-7"]
-        assert [m["display_name"] for m in catalog["models"]] == ["gpt-5.5", "claude-opus-4-7"]
+        assert [m["display_name"] for m in catalog["models"]] == ["GPT-5.5", "claude-opus-4-7"]
         assert [m["priority"] for m in catalog["models"]] == [0, 1]
 
-    def test_every_entry_has_the_fields_codex_requires(self, tmp_path):
+    def _entries(self, codex_home):
+        return {m["slug"]: m for m in json.loads((codex_home / "litellm-models.json").read_text())["models"]}
+
+    def test_known_model_keeps_the_installed_codex_entry(self, tmp_path):
+        self._sync(self._listing(self._row("gpt-5.5", mode="chat")), tmp_path)
+        assert self._entries(tmp_path)["gpt-5.5"] == {**_STOCK_MODELS["gpt-5.5"], "priority": 0}
+
+    def test_hidden_stock_model_is_listed_when_the_proxy_serves_it(self, tmp_path):
+        self._sync(self._listing(self._row("gpt-5.4")), tmp_path)
+        entry = self._entries(tmp_path)["gpt-5.4"]
+        assert entry["visibility"] == "list"
+        assert entry["upgrade"] is None
+        assert entry["supported_reasoning_levels"] == _STOCK_REASONING_LEVELS
+
+    def test_stock_upgrade_nudge_survives_when_its_target_is_listed(self, tmp_path):
+        self._sync(self._listing(self._row("gpt-5.4"), self._row("gpt-5.6-terra")), tmp_path)
+        entries = self._entries(tmp_path)
+        assert entries["gpt-5.4"]["upgrade"] == _STOCK_MODELS["gpt-5.4"]["upgrade"]
+        assert [entries["gpt-5.4"]["priority"], entries["gpt-5.6-terra"]["priority"]] == [0, 1]
+
+    def test_unparseable_stock_catalog_is_reported(self, tmp_path):
+        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=_FakeRun(stock="not json"))
+        assert isinstance(result, ModelSyncSkipped)
+        assert result.reason.startswith("`codex debug models` printed no model catalog: ")
+        assert not (tmp_path / "litellm-models.json").exists()
+
+    def test_unknown_model_gets_the_fields_codex_requires(self, tmp_path):
         _, result = self._sync(self._listing(self._row("m")), tmp_path)
         entry = json.loads((tmp_path / "litellm-models.json").read_text())["models"][0]
 
@@ -435,12 +556,17 @@ class TestCodexModelSync:
             assert nullable in entry and entry[nullable] is None
         assert entry["base_instructions"].startswith("You are a coding agent running in the Codex CLI")
 
-    def test_context_window_comes_from_max_input_tokens(self, tmp_path):
-        listing = self._listing(self._row("big", max_input_tokens=400000), self._row("unknown"))
-        _, result = self._sync(listing, tmp_path)
-        models = {m["slug"]: m for m in json.loads((tmp_path / "litellm-models.json").read_text())["models"]}
+    def test_context_window_comes_from_max_input_tokens_for_unknown_models_only(self, tmp_path):
+        listing = self._listing(
+            self._row("big", max_input_tokens=400000),
+            self._row("unknown"),
+            self._row("gpt-5.5", max_input_tokens=400000),
+        )
+        self._sync(listing, tmp_path)
+        models = self._entries(tmp_path)
         assert models["big"]["context_window"] == 400000
         assert models["unknown"]["context_window"] is None
+        assert models["gpt-5.5"]["context_window"] == 272000
 
     def test_non_chat_models_are_left_out(self, tmp_path):
         listing = self._listing(
@@ -544,7 +670,8 @@ class TestCodexModelSync:
             run=run,
         )
         assert self._catalog_path(result) == str(tmp_path / "litellm-models.json")
-        assert binary in run.calls[0][0]
+        assert len(run.calls) == 2
+        assert all(binary in command for command, _ in run.calls)
 
     def test_opencode_dispatch_never_runs_codex(self):
         def boom(*a, **k):
@@ -561,19 +688,21 @@ class TestCodexModelSync:
         )
         assert "OPENCODE_CONFIG_CONTENT" in result
 
-    def test_catalog_is_read_back_through_codex_before_launch(self, tmp_path):
+    def test_codex_lists_its_own_models_then_reads_the_catalog_back_before_launch(self, tmp_path):
         run = _FakeRun()
         _, result = self._sync(self._listing(self._row("m")), tmp_path, run=run)
         path = self._catalog_path(result)
 
-        assert len(run.calls) == 1
-        command, options = run.calls[0]
-        assert command == ("codex", "-c", f"model_catalog_json={json.dumps(path)}", "debug", "models")
-        assert options["env"] == {"CODEX_HOME": str(tmp_path)}
-        assert options["stdin"] is subprocess.DEVNULL
-        assert options["capture_output"] is True
-        assert options["text"] is True
-        assert options["timeout"] == 10
+        assert [command for command, _ in run.calls] == [
+            ("codex", "debug", "models"),
+            ("codex", "-c", f"model_catalog_json={json.dumps(path)}", "debug", "models"),
+        ]
+        for _, options in run.calls:
+            assert options["env"] == {"CODEX_HOME": str(tmp_path)}
+            assert options["stdin"] is subprocess.DEVNULL
+            assert options["capture_output"] is True
+            assert options["text"] is True
+            assert options["timeout"] == 10
 
     def test_codex_rejecting_the_catalog_skips_the_sync_and_keeps_the_file(self, tmp_path):
         stderr = (
@@ -591,9 +720,12 @@ class TestCodexModelSync:
 
     def test_codex_without_debug_models_skips_the_sync(self, tmp_path):
         stderr = "error: unrecognized subcommand 'models'\n\nUsage: codex debug [OPTIONS] <COMMAND>\n"
-        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=_FakeRun(2, stderr))
+        run = _FakeRun(2, stderr, stock=None)
+        _, result = self._sync(self._listing(self._row("m")), tmp_path, run=run)
         assert isinstance(result, ModelSyncSkipped)
         assert result.reason == "`codex debug models` exited 2: error: unrecognized subcommand 'models'"
+        assert len(run.calls) == 1
+        assert not (tmp_path / "litellm-models.json").exists()
 
     def test_codex_failing_silently_is_reported(self, tmp_path):
         _, result = self._sync(self._listing(self._row("m")), tmp_path, run=_FakeRun(1))
@@ -625,7 +757,10 @@ class TestCodexModelSync:
         )
         override = f"model_catalog_json={json.dumps(self._catalog_path(result))}"
         doubled = override.replace('"', '""')
-        assert run.calls[0][0] == f'{_CMD_PREFIX}""{shim}" "-c" "{doubled}" "debug" "models""'
+        assert [command for command, _ in run.calls] == [
+            f'{_CMD_PREFIX}""{shim}" "debug" "models""',
+            f'{_CMD_PREFIX}""{shim}" "-c" "{doubled}" "debug" "models""',
+        ]
 
     def test_default_binary_is_codex_on_path(self):
         assert _default_of(codex_model_sync_args, "binary") == "codex"

From 15721e52effaa79ad042b23ef91188798a20be43 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 12 Sep 2026 16:50:50 -0700
Subject: [PATCH 10/67] fix(cli): mark proxy-served stock Codex models as
 selectable with an API key

Codex hides catalog entries whose supported_in_api is false when it runs
with an API key, so a stock entry the proxy serves now carries
supported_in_api true alongside its list visibility.
---
 litellm/proxy/client/cli/commands/agents.py        | 9 ++++++---
 tests/test_litellm/proxy/client/cli/test_agents.py | 7 +++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py
index 1422514372a..decb520c3a0 100644
--- a/litellm/proxy/client/cli/commands/agents.py
+++ b/litellm/proxy/client/cli/commands/agents.py
@@ -447,6 +447,7 @@ class _StockCodexModel(BaseModel):
     slug: str
     priority: int
     visibility: str
+    supported_in_api: bool = True
     upgrade: _StockCodexUpgrade | None = None
 
 
@@ -474,7 +475,9 @@ def _codex_catalog_entry(
             base_instructions=instructions,
         )
     upgrade: Final = stock.upgrade if stock.upgrade is not None and stock.upgrade.model in served else None
-    return stock.model_copy(update={"priority": priority, "visibility": "list", "upgrade": upgrade})
+    return stock.model_copy(
+        update={"priority": priority, "visibility": "list", "supported_in_api": True, "upgrade": upgrade}
+    )
 
 
 def codex_model_catalog(
@@ -486,8 +489,8 @@ def codex_model_catalog(
     Passing a catalog replaces Codex's built-in one, so a proxy model the
     installed Codex knows keeps that Codex's own entry and the proxy only
     decides its place in the picker: the listing orders it, lists it even when
-    Codex hides it, and keeps Codex's upgrade nudge only when the model it
-    points at is served too. A model Codex does not know gets the fallback
+    Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
+    when the model it points at is served too. A model Codex does not know gets the fallback
     entry, with the same base instructions Codex itself uses so the agent never
     runs without a system prompt.
     """
diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py
index cc7c3a14f44..c437f4c12c5 100644
--- a/tests/test_litellm/proxy/client/cli/test_agents.py
+++ b/tests/test_litellm/proxy/client/cli/test_agents.py
@@ -527,6 +527,13 @@ class TestCodexModelSync:
         assert entry["upgrade"] is None
         assert entry["supported_reasoning_levels"] == _STOCK_REASONING_LEVELS
 
+    def test_api_disabled_stock_model_is_selectable_when_the_proxy_serves_it(self, tmp_path):
+        self._sync(self._listing(self._row("codex-auto-review")), tmp_path)
+        entry = self._entries(tmp_path)["codex-auto-review"]
+        assert entry["supported_in_api"] is True
+        assert entry["visibility"] == "list"
+        assert entry["base_instructions"] == _STOCK_MODELS["codex-auto-review"]["base_instructions"]
+
     def test_stock_upgrade_nudge_survives_when_its_target_is_listed(self, tmp_path):
         self._sync(self._listing(self._row("gpt-5.4"), self._row("gpt-5.6-terra")), tmp_path)
         entries = self._entries(tmp_path)

From c15f3e92289125e02d1f351d074c37c4241791e2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 12 Sep 2026 17:29:09 -0700
Subject: [PATCH 11/67] fix(cli): decode codex debug models output as UTF-8

---
 litellm/proxy/client/cli/commands/agents.py    |  2 +-
 .../proxy/client/cli/test_agents.py            | 18 +++++++++++++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/client/cli/commands/agents.py b/litellm/proxy/client/cli/commands/agents.py
index decb520c3a0..15b111ff016 100644
--- a/litellm/proxy/client/cli/commands/agents.py
+++ b/litellm/proxy/client/cli/commands/agents.py
@@ -546,7 +546,7 @@ def _codex_debug_models(
             env=dict(env),
             stdin=subprocess.DEVNULL,
             capture_output=True,
-            text=True,
+            encoding="utf-8",
             timeout=_CODEX_PREFLIGHT_TIMEOUT_SECONDS,
         )
     except (OSError, subprocess.TimeoutExpired) as e:
diff --git a/tests/test_litellm/proxy/client/cli/test_agents.py b/tests/test_litellm/proxy/client/cli/test_agents.py
index c437f4c12c5..f6f1c2fec3b 100644
--- a/tests/test_litellm/proxy/client/cli/test_agents.py
+++ b/tests/test_litellm/proxy/client/cli/test_agents.py
@@ -540,6 +540,22 @@ class TestCodexModelSync:
         assert entries["gpt-5.4"]["upgrade"] == _STOCK_MODELS["gpt-5.4"]["upgrade"]
         assert [entries["gpt-5.4"]["priority"], entries["gpt-5.6-terra"]["priority"]] == [0, 1]
 
+    def test_stock_catalog_is_decoded_as_utf8_regardless_of_locale(self, tmp_path):
+        description = "Modelo equilibrado para el trabajo diario, con acentos y ñ."
+        catalog = {"models": [{**_STOCK_MODELS["gpt-5.5"], "description": description}]}
+        stock = json.dumps(catalog, ensure_ascii=False).encode("utf-8")
+
+        def locale_bound_run(args, **kwargs):
+            if "model_catalog_json=" in str(args):
+                return subprocess.CompletedProcess(args, 0, "", "")
+            return subprocess.CompletedProcess(args, 0, stock.decode(kwargs.get("encoding") or "ascii"), "")
+
+        _, result = self._sync(self._listing(self._row("gpt-5.5")), tmp_path, run=locale_bound_run)
+
+        assert isinstance(result, ModelSyncArgs)
+        written = json.loads((tmp_path / "litellm-models.json").read_text(encoding="utf-8"))["models"]
+        assert [m["description"] for m in written] == [description]
+
     def test_unparseable_stock_catalog_is_reported(self, tmp_path):
         _, result = self._sync(self._listing(self._row("m")), tmp_path, run=_FakeRun(stock="not json"))
         assert isinstance(result, ModelSyncSkipped)
@@ -708,7 +724,7 @@ class TestCodexModelSync:
             assert options["env"] == {"CODEX_HOME": str(tmp_path)}
             assert options["stdin"] is subprocess.DEVNULL
             assert options["capture_output"] is True
-            assert options["text"] is True
+            assert options["encoding"] == "utf-8"
             assert options["timeout"] == 10
 
     def test_codex_rejecting_the_catalog_skips_the_sync_and_keeps_the_file(self, tmp_path):

From 299cd084f9e90e2c4e31024e45a599fdb0395773 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 12 Sep 2026 20:29:16 -0700
Subject: [PATCH 12/67] refactor(prompt_templates): share the tool use id
 sanitizer between the Anthropic and Bedrock paths

---
 .../prompt_templates/factory.py               | 29 +++++++++----------
 ...llm_core_utils_prompt_templates_factory.py | 17 +++++++++++
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py
index f7f4a964c9b..d61c3235c5e 100644
--- a/litellm/litellm_core_utils/prompt_templates/factory.py
+++ b/litellm/litellm_core_utils/prompt_templates/factory.py
@@ -1500,32 +1500,29 @@ def convert_to_gemini_tool_call_result(
     return _part
 
 
-def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
-    """
-    Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$
-
-    Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens.
-    This function replaces any invalid characters with underscores.
-    """
-    # Replace any character that's not alphanumeric, underscore, or hyphen with underscore
-    sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id)
-    # Ensure it's not empty (fallback to a default if needed)
-    if not sanitized:
-        sanitized = "tool_use_id"
-    return sanitized
-
-
+_TOOL_USE_ID_FALLBACK: Final = "tool_use_id"
+_ANTHROPIC_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]")
+_BEDROCK_TOOL_USE_ID_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_.:-]")
 _BEDROCK_TOOL_USE_ID_MAX_LEN: Final = 64
 _BEDROCK_TOOL_USE_ID_HASH_LEN: Final = 8
 
 
+def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Pattern[str]) -> str:
+    return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK
+
+
+def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
+    """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$."""
+    return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS)
+
+
 def _sanitize_bedrock_tool_use_id(tool_use_id: str) -> str:
     """
     Bedrock Converse requires toolUseId to match [a-zA-Z0-9_.:-]+ and be at most 64 chars.
     Ids that need rewriting get a short hash of the original appended so two ids that only
     differ in a replaced char or past the cut still map to distinct values.
     """
-    sanitized: Final = re.sub(r"[^a-zA-Z0-9_.:-]", "_", tool_use_id) or "tool_use_id"
+    sanitized: Final = _replace_invalid_tool_use_id_chars(tool_use_id, _BEDROCK_TOOL_USE_ID_INVALID_CHARS)
     if sanitized == tool_use_id and len(sanitized) <= _BEDROCK_TOOL_USE_ID_MAX_LEN:
         return sanitized
     digest: Final = hashlib.sha256(tool_use_id.encode()).hexdigest()[:_BEDROCK_TOOL_USE_ID_HASH_LEN]
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
index fe8a9bd5205..034062826f6 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
@@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     _convert_to_bedrock_tool_call_invoke,
     _convert_to_bedrock_tool_call_result,
     anthropic_messages_pt,
+    convert_to_anthropic_tool_result,
     convert_to_gemini_tool_call_result,
     make_valid_bedrock_tool_name,
     ollama_pt,
@@ -2219,6 +2220,7 @@ _BEDROCK_TOOL_USE_ID_RE = re.compile(r"^[a-zA-Z0-9_.:-]{1,64}$")
         "call|with|pipes",
         "call_" + "y" * 60 + "|end",
         "call:ok.dots-and_under",
+        "",
     ],
 )
 def test_bedrock_tool_use_id_is_sanitized_consistently_for_invoke_and_result(tool_call_id):
@@ -2291,6 +2293,21 @@ def test_bedrock_tool_call_invoke_concatenated_json_long_id_stays_within_limit()
     assert all(_BEDROCK_TOOL_USE_ID_RE.match(i) for i in ids)
 
 
+@pytest.mark.parametrize(
+    ("tool_call_id", "expected"),
+    [
+        ("call|with|pipes", "call_with_pipes"),
+        ("call:ok.dots", "call_ok_dots"),
+        ("call_" + "x" * 100, "call_" + "x" * 100),
+        ("toolu_01AbC-xyz", "toolu_01AbC-xyz"),
+        ("", "tool_use_id"),
+    ],
+)
+def test_anthropic_tool_use_id_keeps_pattern_only_rewrite_with_no_cap_or_hash(tool_call_id, expected):
+    result = convert_to_anthropic_tool_result({"role": "tool", "tool_call_id": tool_call_id, "content": "ok"})
+    assert result["tool_use_id"] == expected
+
+
 def test_bedrock_tool_call_invoke_concatenated_json():
     """
     Tool call whose arguments contain multiple concatenated JSON objects

From 8607c49ea1877f28b6586be9c2e7f2be95391982 Mon Sep 17 00:00:00 2001
From: IToSSc <IToSSc@users.noreply.github.com>
Date: Tue, 15 Sep 2026 14:16:20 +0800
Subject: [PATCH 13/67] feat: add aihubmix provider pricing entries

Add 72 model price entries for the aihubmix openai_like provider so
cost tracking and budgets work for aihubmix/* model calls. The
provider is already registered in llms/openai_like/providers.json
but model_prices_and_context_window.json had zero entries for it.

The Anthropic-family entries (claude-fable-5, claude-haiku-4-5,
claude-opus-4-8, claude-opus-5, claude-sonnet-5) carry the same
supports_adaptive_thinking, thinking_always_on,
supports_sampling_params, and prompt_cache_min_tokens flags already
used by this repo's other Anthropic re-exports (azure_ai, databricks,
openrouter, and so on) for the same underlying models, since those
flags gate request shapes the provider otherwise rejects with a 400.

TASK-2BK38Y
---
 ...odel_prices_and_context_window_backup.json | 1125 +++++++++++++++++
 model_prices_and_context_window.json          | 1125 +++++++++++++++++
 2 files changed, 2250 insertions(+)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 9f91cf82f41..c870e9c2255 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -66038,5 +66038,1130 @@
         "mode": "chat",
         "output_cost_per_token": 8e-06,
         "source": "https://api.together.ai/v1/models"
+    },
+    "aihubmix/agnes-2.5-flash": {
+        "input_cost_per_token": 3e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 512000,
+        "max_output_tokens": 65500,
+        "max_tokens": 65500,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/agnes-2.5-pro": {
+        "cache_read_input_token_cost": 3.78e-09,
+        "input_cost_per_token": 4.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 9e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/cc-glm-5.1": {
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 2.2e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/claude-fable-5": {
+        "cache_read_input_token_cost": 1.1e-06,
+        "cache_creation_input_token_cost": 1.375e-05,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_sampling_params": false
+    },
+    "aihubmix/claude-haiku-4-5": {
+        "cache_read_input_token_cost": 1.1e-07,
+        "cache_creation_input_token_cost": 1.375e-06,
+        "input_cost_per_token": 1.1e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 4096
+    },
+    "aihubmix/claude-opus-4-8-think": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_creation_input_token_cost": 6.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 1024,
+        "supports_adaptive_thinking": true,
+        "supports_sampling_params": false
+    },
+    "aihubmix/claude-opus-5": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_creation_input_token_cost": 6.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_adaptive_thinking": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_sampling_params": false
+    },
+    "aihubmix/claude-sonnet-5": {
+        "cache_read_input_token_cost": 2e-07,
+        "cache_creation_input_token_cost": 2.5e-06,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 1024,
+        "supports_adaptive_thinking": true,
+        "supports_sampling_params": false
+    },
+    "aihubmix/coding-glm-5.3": {
+        "cache_read_input_token_cost": 1.5e-08,
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 2.2e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/coding-kimi-k3": {
+        "cache_read_input_token_cost": 6.6e-08,
+        "input_cost_per_token": 4.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.61333e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/coding-xiaomi-mimo-v2-omni": {
+        "cache_read_input_token_cost": 1.6e-08,
+        "input_cost_per_token": 8e-08,
+        "litellm_provider": "aihubmix",
+        "mode": "chat",
+        "output_cost_per_token": 4e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/coding-xiaomi-mimo-v2.5": {
+        "cache_read_input_token_cost": 1.6e-09,
+        "input_cost_per_token": 8e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.6e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/coding-xiaomi-mimo-v2.5-pro": {
+        "cache_read_input_token_cost": 1.6e-09,
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 4e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/command-a-plus-05-2026": {
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 1e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/deepseek-v4-flash": {
+        "cache_read_input_token_cost": 2.84e-08,
+        "input_cost_per_token": 1.42e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 384000,
+        "max_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 2.84e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/deepseek-v4-pro": {
+        "cache_read_input_token_cost": 1.4027e-07,
+        "input_cost_per_token": 1.69e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 384000,
+        "max_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 3.38e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/doubao-seed-2-0-code-preview": {
+        "cache_read_input_token_cost": 9.644e-08,
+        "input_cost_per_token": 4.822e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.411e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-0-lite-260428": {
+        "cache_read_input_token_cost": 1.8082e-08,
+        "input_cost_per_token": 9.041e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.4246e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-0-mini": {
+        "cache_read_input_token_cost": 6.027e-09,
+        "input_cost_per_token": 3.0136e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3.0136e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-0-pro": {
+        "cache_read_input_token_cost": 9.644e-08,
+        "input_cost_per_token": 4.822e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.411e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-1-turbo": {
+        "cache_read_input_token_cost": 9.295e-08,
+        "input_cost_per_token": 4.6475e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 2.32375e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/ernie-5.1": {
+        "cache_read_input_token_cost": 5.634e-07,
+        "input_cost_per_token": 5.634e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 119000,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 2.5353e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true
+    },
+    "aihubmix/gemini-3-flash-preview": {
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3-flash-preview-search": {
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.1-pro-preview": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.1-pro-preview-customtools": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 2.499999e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.7-flash": {
+        "cache_read_input_token_cost": 7.5e-08,
+        "input_cost_per_token": 7.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 3.75e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemma-4-26b-a4b-it": {
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131100,
+        "max_tokens": 131100,
+        "mode": "chat",
+        "output_cost_per_token": 3.9998e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/gemma-4-31b-it": {
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131100,
+        "max_tokens": 131100,
+        "mode": "chat",
+        "output_cost_per_token": 3.9998e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/glm-5.2-fast-preview": {
+        "cache_read_input_token_cost": 5.635e-07,
+        "input_cost_per_token": 2.254e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 7.889e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/glm-5.3": {
+        "cache_read_input_token_cost": 2.817e-07,
+        "input_cost_per_token": 1.1268e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.9438e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/glm-5.3-flash": {
+        "cache_read_input_token_cost": 2.817e-08,
+        "input_cost_per_token": 1.1268e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.9438e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/glm-5v-turbo": {
+        "cache_read_input_token_cost": 1.69008e-07,
+        "input_cost_per_token": 7.042e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.09848e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.3-codex": {
+        "cache_read_input_token_cost": 1.75e-07,
+        "input_cost_per_token": 1.75e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.4-high": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.4-low": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.4-mini": {
+        "cache_read_input_token_cost": 7.5e-08,
+        "input_cost_per_token": 7.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 4.5e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.4-nano": {
+        "cache_read_input_token_cost": 2e-08,
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.25e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.5": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.5-pro": {
+        "input_cost_per_token": 3e-05,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 0.00018,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.6-luna": {
+        "cache_read_input_token_cost": 2e-08,
+        "cache_creation_input_token_cost": 2.5e-07,
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.6-sol-disc": {
+        "cache_read_input_token_cost": 4e-07,
+        "cache_creation_input_token_cost": 5e-06,
+        "input_cost_per_token": 4e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.6-terra": {
+        "cache_read_input_token_cost": 2e-07,
+        "cache_creation_input_token_cost": 2.5e-06,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-chat-latest": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/grok-4-20-non-reasoning": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/grok-4-20-reasoning": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/grok-4.6": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/grok-build-0.1": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 1e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 2e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/hy3": {
+        "cache_read_input_token_cost": 3.905e-08,
+        "input_cost_per_token": 1.562e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6.248e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/hy4-preview": {
+        "cache_read_input_token_cost": 4.225e-08,
+        "input_cost_per_token": 8.45e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 2.535e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/kimi-k2.6": {
+        "cache_read_input_token_cost": 1.60835e-07,
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 3.9995e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/kimi-k2.7-code-highspeed": {
+        "cache_read_input_token_cost": 3.2167e-07,
+        "input_cost_per_token": 1.9e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 7.999e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/kimi-k3": {
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/longcat-2.0": {
+        "cache_read_input_token_cost": 1.5492e-08,
+        "input_cost_per_token": 7.746e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.0984e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true
+    },
+    "aihubmix/mai-thinking-1": {
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 8e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/mimo-v2-omni": {
+        "cache_read_input_token_cost": 8.8e-08,
+        "input_cost_per_token": 4.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 2.2e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/mimo-v2-pro": {
+        "cache_read_input_token_cost": 2.2e-07,
+        "input_cost_per_token": 1.1e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3.3e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_web_search": true
+    },
+    "aihubmix/minimax-m2.7": {
+        "cache_read_input_token_cost": 5.916e-08,
+        "input_cost_per_token": 2.958e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 204800,
+        "max_output_tokens": 204800,
+        "max_tokens": 204800,
+        "mode": "chat",
+        "output_cost_per_token": 1.1832e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/minimax-m3": {
+        "input_cost_per_token": 2.88e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.152e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/muse-spark-1.2": {
+        "input_cost_per_token": 1.375e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 4.675e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/qwen3-coder-next": {
+        "input_cost_per_token": 1.37e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 5.48e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/qwen3.5-122b-a10b": {
+        "input_cost_per_token": 1.126e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 9.008e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.5-397b-a17b": {
+        "input_cost_per_token": 1.644e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 9.864e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.6-27b": {
+        "input_cost_per_token": 4.22e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 2.532e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.6-35b-a3b": {
+        "input_cost_per_token": 2.54e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 1.524e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.6-max-preview": {
+        "cache_read_input_token_cost": 1.268e-07,
+        "cache_creation_input_token_cost": 1.585e-06,
+        "input_cost_per_token": 1.268e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 7.608e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.7-plus": {
+        "cache_read_input_token_cost": 5.64e-08,
+        "cache_creation_input_token_cost": 3.525e-07,
+        "input_cost_per_token": 2.82e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.128e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.8-2.4t-a95b": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.8-flash": {
+        "cache_read_input_token_cost": 1.4075e-08,
+        "cache_creation_input_token_cost": 1.75937e-07,
+        "input_cost_per_token": 1.126e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.80025e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.8-max": {
+        "cache_read_input_token_cost": 1.69e-07,
+        "cache_creation_input_token_cost": 2.1125e-06,
+        "input_cost_per_token": 1.69e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 5.07e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/step-3.7-flash": {
+        "cache_read_input_token_cost": 4.4e-08,
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true
     }
 }
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 9f91cf82f41..c870e9c2255 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -66038,5 +66038,1130 @@
         "mode": "chat",
         "output_cost_per_token": 8e-06,
         "source": "https://api.together.ai/v1/models"
+    },
+    "aihubmix/agnes-2.5-flash": {
+        "input_cost_per_token": 3e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 512000,
+        "max_output_tokens": 65500,
+        "max_tokens": 65500,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/agnes-2.5-pro": {
+        "cache_read_input_token_cost": 3.78e-09,
+        "input_cost_per_token": 4.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 9e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/cc-glm-5.1": {
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 2.2e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/claude-fable-5": {
+        "cache_read_input_token_cost": 1.1e-06,
+        "cache_creation_input_token_cost": 1.375e-05,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
+        "supports_sampling_params": false
+    },
+    "aihubmix/claude-haiku-4-5": {
+        "cache_read_input_token_cost": 1.1e-07,
+        "cache_creation_input_token_cost": 1.375e-06,
+        "input_cost_per_token": 1.1e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 4096
+    },
+    "aihubmix/claude-opus-4-8-think": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_creation_input_token_cost": 6.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 1024,
+        "supports_adaptive_thinking": true,
+        "supports_sampling_params": false
+    },
+    "aihubmix/claude-opus-5": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_creation_input_token_cost": 6.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_adaptive_thinking": true,
+        "prompt_cache_min_tokens": 512,
+        "supports_sampling_params": false
+    },
+    "aihubmix/claude-sonnet-5": {
+        "cache_read_input_token_cost": 2e-07,
+        "cache_creation_input_token_cost": 2.5e-06,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "prompt_cache_min_tokens": 1024,
+        "supports_adaptive_thinking": true,
+        "supports_sampling_params": false
+    },
+    "aihubmix/coding-glm-5.3": {
+        "cache_read_input_token_cost": 1.5e-08,
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 2.2e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/coding-kimi-k3": {
+        "cache_read_input_token_cost": 6.6e-08,
+        "input_cost_per_token": 4.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.61333e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/coding-xiaomi-mimo-v2-omni": {
+        "cache_read_input_token_cost": 1.6e-08,
+        "input_cost_per_token": 8e-08,
+        "litellm_provider": "aihubmix",
+        "mode": "chat",
+        "output_cost_per_token": 4e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/coding-xiaomi-mimo-v2.5": {
+        "cache_read_input_token_cost": 1.6e-09,
+        "input_cost_per_token": 8e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.6e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/coding-xiaomi-mimo-v2.5-pro": {
+        "cache_read_input_token_cost": 1.6e-09,
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 4e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/command-a-plus-05-2026": {
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 1e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/deepseek-v4-flash": {
+        "cache_read_input_token_cost": 2.84e-08,
+        "input_cost_per_token": 1.42e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 384000,
+        "max_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 2.84e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/deepseek-v4-pro": {
+        "cache_read_input_token_cost": 1.4027e-07,
+        "input_cost_per_token": 1.69e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 384000,
+        "max_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 3.38e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/doubao-seed-2-0-code-preview": {
+        "cache_read_input_token_cost": 9.644e-08,
+        "input_cost_per_token": 4.822e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.411e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-0-lite-260428": {
+        "cache_read_input_token_cost": 1.8082e-08,
+        "input_cost_per_token": 9.041e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.4246e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-0-mini": {
+        "cache_read_input_token_cost": 6.027e-09,
+        "input_cost_per_token": 3.0136e-08,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3.0136e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-0-pro": {
+        "cache_read_input_token_cost": 9.644e-08,
+        "input_cost_per_token": 4.822e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2.411e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/doubao-seed-2-1-turbo": {
+        "cache_read_input_token_cost": 9.295e-08,
+        "input_cost_per_token": 4.6475e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 2.32375e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/ernie-5.1": {
+        "cache_read_input_token_cost": 5.634e-07,
+        "input_cost_per_token": 5.634e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 119000,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 2.5353e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true
+    },
+    "aihubmix/gemini-3-flash-preview": {
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3-flash-preview-search": {
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.1-pro-preview": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.1-pro-preview-customtools": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.5-flash-lite": {
+        "cache_read_input_token_cost": 3e-08,
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 2.499999e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemini-3.7-flash": {
+        "cache_read_input_token_cost": 7.5e-08,
+        "input_cost_per_token": 7.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 3.75e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gemma-4-26b-a4b-it": {
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131100,
+        "max_tokens": 131100,
+        "mode": "chat",
+        "output_cost_per_token": 3.9998e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/gemma-4-31b-it": {
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131100,
+        "max_tokens": 131100,
+        "mode": "chat",
+        "output_cost_per_token": 3.9998e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/glm-5.2-fast-preview": {
+        "cache_read_input_token_cost": 5.635e-07,
+        "input_cost_per_token": 2.254e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 7.889e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/glm-5.3": {
+        "cache_read_input_token_cost": 2.817e-07,
+        "input_cost_per_token": 1.1268e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.9438e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/glm-5.3-flash": {
+        "cache_read_input_token_cost": 2.817e-08,
+        "input_cost_per_token": 1.1268e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.9438e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/glm-5v-turbo": {
+        "cache_read_input_token_cost": 1.69008e-07,
+        "input_cost_per_token": 7.042e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.09848e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.3-codex": {
+        "cache_read_input_token_cost": 1.75e-07,
+        "input_cost_per_token": 1.75e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.4e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.4-high": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.4-low": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.4-mini": {
+        "cache_read_input_token_cost": 7.5e-08,
+        "input_cost_per_token": 7.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 4.5e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.4-nano": {
+        "cache_read_input_token_cost": 2e-08,
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.25e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.5": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.5-pro": {
+        "input_cost_per_token": 3e-05,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 0.00018,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/gpt-5.6-luna": {
+        "cache_read_input_token_cost": 2e-08,
+        "cache_creation_input_token_cost": 2.5e-07,
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.6-sol-disc": {
+        "cache_read_input_token_cost": 4e-07,
+        "cache_creation_input_token_cost": 5e-06,
+        "input_cost_per_token": 4e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-5.6-terra": {
+        "cache_read_input_token_cost": 2e-07,
+        "cache_creation_input_token_cost": 2.5e-06,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1050000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/gpt-chat-latest": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/grok-4-20-non-reasoning": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/grok-4-20-reasoning": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/grok-4.6": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/grok-build-0.1": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 1e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 256000,
+        "max_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 2e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/hy3": {
+        "cache_read_input_token_cost": 3.905e-08,
+        "input_cost_per_token": 1.562e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6.248e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/hy4-preview": {
+        "cache_read_input_token_cost": 4.225e-08,
+        "input_cost_per_token": 8.45e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 2.535e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/kimi-k2.6": {
+        "cache_read_input_token_cost": 1.60835e-07,
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 3.9995e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/kimi-k2.7-code-highspeed": {
+        "cache_read_input_token_cost": 3.2167e-07,
+        "input_cost_per_token": 1.9e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 7.999e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/kimi-k3": {
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/longcat-2.0": {
+        "cache_read_input_token_cost": 1.5492e-08,
+        "input_cost_per_token": 7.746e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.0984e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true
+    },
+    "aihubmix/mai-thinking-1": {
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 8e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/mimo-v2-omni": {
+        "cache_read_input_token_cost": 8.8e-08,
+        "input_cost_per_token": 4.4e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 2.2e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/mimo-v2-pro": {
+        "cache_read_input_token_cost": 2.2e-07,
+        "input_cost_per_token": 1.1e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3.3e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_web_search": true
+    },
+    "aihubmix/minimax-m2.7": {
+        "cache_read_input_token_cost": 5.916e-08,
+        "input_cost_per_token": 2.958e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 204800,
+        "max_output_tokens": 204800,
+        "max_tokens": 204800,
+        "mode": "chat",
+        "output_cost_per_token": 1.1832e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/minimax-m3": {
+        "input_cost_per_token": 2.88e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.152e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true
+    },
+    "aihubmix/muse-spark-1.2": {
+        "input_cost_per_token": 1.375e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 4.675e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true
+    },
+    "aihubmix/qwen3-coder-next": {
+        "input_cost_per_token": 1.37e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 5.48e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_response_schema": true
+    },
+    "aihubmix/qwen3.5-122b-a10b": {
+        "input_cost_per_token": 1.126e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 9.008e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.5-397b-a17b": {
+        "input_cost_per_token": 1.644e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 9.864e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.6-27b": {
+        "input_cost_per_token": 4.22e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 2.532e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.6-35b-a3b": {
+        "input_cost_per_token": 2.54e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 1.524e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.6-max-preview": {
+        "cache_read_input_token_cost": 1.268e-07,
+        "cache_creation_input_token_cost": 1.585e-06,
+        "input_cost_per_token": 1.268e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 7.608e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.7-plus": {
+        "cache_read_input_token_cost": 5.64e-08,
+        "cache_creation_input_token_cost": 3.525e-07,
+        "input_cost_per_token": 2.82e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.128e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.8-2.4t-a95b": {
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.8-flash": {
+        "cache_read_input_token_cost": 1.4075e-08,
+        "cache_creation_input_token_cost": 1.75937e-07,
+        "input_cost_per_token": 1.126e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 3.80025e-07,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/qwen3.8-max": {
+        "cache_read_input_token_cost": 1.69e-07,
+        "cache_creation_input_token_cost": 2.1125e-06,
+        "input_cost_per_token": 1.69e-06,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 5.07e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "aihubmix/step-3.7-flash": {
+        "cache_read_input_token_cost": 4.4e-08,
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "aihubmix",
+        "max_input_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-06,
+        "source": "https://aihubmix.com/api/v1/models",
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_vision": true
     }
 }

From bc17459548b1a14bd6856d42e1339a027f105796 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 15 Sep 2026 07:46:41 +0000
Subject: [PATCH 14/67] fix(proxy): include litellm_call_id in LLM API
 exception logs

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/common_request_processing.py    | 19 +++--
 .../proxy/test_common_request_processing.py   | 70 ++++++++++++++++++-
 2 files changed, 84 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 4a4daa68cce..a29079433e9 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -1451,10 +1451,12 @@ def _has_attribute_error_in_chain(exc: Exception) -> bool:
 _CLIENT_DISCONNECT_DETAIL: Final = "Client disconnected the request"
 
 
-def _log_llm_api_exception(e: Exception) -> None:
+def _log_llm_api_exception(e: Exception, litellm_call_id: str | None) -> None:
     if getattr(e, "status_code", None) == 499 and getattr(e, "detail", None) == _CLIENT_DISCONNECT_DETAIL:
         verbose_proxy_logger.info(
-            "litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, upstream LLM request cancelled"
+            "litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, "
+            "upstream LLM request cancelled - litellm_call_id=%s",
+            litellm_call_id,
         )
         return
     log_fn: Final = (
@@ -1462,7 +1464,12 @@ def _log_llm_api_exception(e: Exception) -> None:
         if is_expected_client_error(e) and not litellm.log_client_error_tracebacks
         else verbose_proxy_logger.exception
     )
-    log_fn("litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - %s", e)
+    log_fn(
+        "litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - litellm_call_id=%s - %s",
+        litellm_call_id,
+        e,
+        extra=MappingProxyType({"litellm_call_id": litellm_call_id}),
+    )
 
 
 async def _cancel_llm_call_on_client_disconnect(
@@ -3421,7 +3428,11 @@ class ProxyBaseLLMRequestProcessing:
         version: str | None = None,
     ):
         """Raises ProxyException (OpenAI API compatible) if an exception is raised"""
-        _log_llm_api_exception(e)
+        logging_obj: Final[LiteLLMLoggingObj | None] = self.data.get("litellm_logging_obj", None)
+        _log_llm_api_exception(
+            e,
+            logging_obj.litellm_call_id if logging_obj is not None else self.data.get("litellm_call_id"),
+        )
         # Allow callbacks to transform the error response
         transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
             user_api_key_dict=user_api_key_dict,
diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py
index cabfcc9918f..98e9d30493a 100644
--- a/tests/test_litellm/proxy/test_common_request_processing.py
+++ b/tests/test_litellm/proxy/test_common_request_processing.py
@@ -8169,7 +8169,7 @@ def test_log_llm_api_exception_traceback_only_for_unexpected_errors(exc, expect_
             try:
                 raise exc
             except Exception as raised:
-                _log_llm_api_exception(raised)
+                _log_llm_api_exception(raised, "call-id-for-traceback-test")
     finally:
         verbose_proxy_logger.propagate = False
 
@@ -8663,3 +8663,71 @@ class TestBackgroundResponseRetrievalGovernance:
 
         assert "_guardrail_pipelines" not in data["litellm_metadata"]
         assert "applied_policies" not in data["litellm_metadata"]
+
+
+class TestErrorLogCarriesCallId:
+    """Regression for LIT-5856 / #37532: the ERROR line emitted for a failed LLM
+    request must carry the litellm_call_id the client got back in the
+    x-litellm-call-id response header, so a logged exception can be tied to a
+    specific request."""
+
+    async def _invoke(self, data: dict) -> None:
+        from litellm._logging import verbose_proxy_logger
+
+        processor: Final = ProxyBaseLLMRequestProcessing(data=data)
+        proxy_logging_obj: Final = MagicMock()
+        proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
+        verbose_proxy_logger.propagate = True
+        try:
+            with pytest.raises(ProxyException):
+                await processor._handle_llm_api_exception(
+                    e=ValueError("upstream blew up"),
+                    user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"),
+                    proxy_logging_obj=proxy_logging_obj,
+                )
+        finally:
+            verbose_proxy_logger.propagate = False
+
+    @staticmethod
+    def _error_record(caplog: pytest.LogCaptureFixture):
+        return next(r for r in caplog.records if "_handle_llm_api_exception(): Exception occured" in r.getMessage())
+
+    async def test_call_id_from_logging_obj_is_logged(self, caplog: pytest.LogCaptureFixture) -> None:
+        call_id: Final = str(uuid.uuid4())
+        logging_obj: Final = MagicMock()
+        logging_obj.litellm_call_id = call_id
+        with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
+            await self._invoke({"litellm_logging_obj": logging_obj, "litellm_call_id": "stale-id"})
+
+        record: Final = self._error_record(caplog)
+        assert record.litellm_call_id == call_id
+        assert call_id in record.getMessage()
+
+    async def test_call_id_falls_back_to_request_data(self, caplog: pytest.LogCaptureFixture) -> None:
+        call_id: Final = str(uuid.uuid4())
+        with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
+            await self._invoke({"litellm_call_id": call_id})
+
+        record: Final = self._error_record(caplog)
+        assert record.litellm_call_id == call_id
+        assert call_id in record.getMessage()
+
+    def test_client_disconnect_log_carries_call_id(self, caplog: pytest.LogCaptureFixture) -> None:
+        from litellm._logging import verbose_proxy_logger
+        from litellm.proxy.common_request_processing import (
+            _CLIENT_DISCONNECT_DETAIL,
+            _log_llm_api_exception,
+        )
+
+        call_id: Final = str(uuid.uuid4())
+        verbose_proxy_logger.propagate = True
+        try:
+            with caplog.at_level("INFO", logger="LiteLLM Proxy"):
+                _log_llm_api_exception(
+                    HTTPException(status_code=499, detail=_CLIENT_DISCONNECT_DETAIL),
+                    call_id,
+                )
+        finally:
+            verbose_proxy_logger.propagate = False
+
+        assert call_id in caplog.records[-1].getMessage()

From 0a8eb56ba40eeceba26bdef6ae61a553b28681ec Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 15 Sep 2026 08:03:45 +0000
Subject: [PATCH 15/67] fix(proxy): fall back to request data when logging
 object has no call id

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/common_request_processing.py          |  2 +-
 .../proxy/test_common_request_processing.py         | 13 ++++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index a29079433e9..2cfcce2c11c 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -3431,7 +3431,7 @@ class ProxyBaseLLMRequestProcessing:
         logging_obj: Final[LiteLLMLoggingObj | None] = self.data.get("litellm_logging_obj", None)
         _log_llm_api_exception(
             e,
-            logging_obj.litellm_call_id if logging_obj is not None else self.data.get("litellm_call_id"),
+            (logging_obj.litellm_call_id if logging_obj is not None else None) or self.data.get("litellm_call_id"),
         )
         # Allow callbacks to transform the error response
         transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py
index 98e9d30493a..cdf2118a1c5 100644
--- a/tests/test_litellm/proxy/test_common_request_processing.py
+++ b/tests/test_litellm/proxy/test_common_request_processing.py
@@ -8671,7 +8671,7 @@ class TestErrorLogCarriesCallId:
     x-litellm-call-id response header, so a logged exception can be tied to a
     specific request."""
 
-    async def _invoke(self, data: dict) -> None:
+    async def _invoke(self, data: dict[str, object]) -> None:
         from litellm._logging import verbose_proxy_logger
 
         processor: Final = ProxyBaseLLMRequestProcessing(data=data)
@@ -8712,6 +8712,17 @@ class TestErrorLogCarriesCallId:
         assert record.litellm_call_id == call_id
         assert call_id in record.getMessage()
 
+    async def test_call_id_falls_back_when_logging_obj_has_none(self, caplog: pytest.LogCaptureFixture) -> None:
+        call_id: Final = str(uuid.uuid4())
+        logging_obj: Final = MagicMock()
+        logging_obj.litellm_call_id = None
+        with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
+            await self._invoke({"litellm_logging_obj": logging_obj, "litellm_call_id": call_id})
+
+        record: Final = self._error_record(caplog)
+        assert record.litellm_call_id == call_id
+        assert call_id in record.getMessage()
+
     def test_client_disconnect_log_carries_call_id(self, caplog: pytest.LogCaptureFixture) -> None:
         from litellm._logging import verbose_proxy_logger
         from litellm.proxy.common_request_processing import (

From 1b474b075f1b70f2dc7e88490e209738bfc6a6fb Mon Sep 17 00:00:00 2001
From: yucheng <yucheng@berri.ai>
Date: Tue, 15 Sep 2026 08:15:14 +0000
Subject: [PATCH 16/67] fix(proxy): attach litellm_call_id to client disconnect
 log record

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/common_request_processing.py                 | 1 +
 tests/test_litellm/proxy/test_common_request_processing.py | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 2cfcce2c11c..37c1ab39632 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -1457,6 +1457,7 @@ def _log_llm_api_exception(e: Exception, litellm_call_id: str | None) -> None:
             "litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, "
             "upstream LLM request cancelled - litellm_call_id=%s",
             litellm_call_id,
+            extra=MappingProxyType({"litellm_call_id": litellm_call_id}),
         )
         return
     log_fn: Final = (
diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py
index cdf2118a1c5..735b0dee3dc 100644
--- a/tests/test_litellm/proxy/test_common_request_processing.py
+++ b/tests/test_litellm/proxy/test_common_request_processing.py
@@ -8741,4 +8741,6 @@ class TestErrorLogCarriesCallId:
         finally:
             verbose_proxy_logger.propagate = False
 
-        assert call_id in caplog.records[-1].getMessage()
+        record: Final = caplog.records[-1]
+        assert record.litellm_call_id == call_id
+        assert call_id in record.getMessage()

From fb00567e4cd275d5cbe8333cb58ac8123c2b3787 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 18:38:07 +0000
Subject: [PATCH 17/67] fix(proxy): track per-member organization spend so the
 Organizations UI shows member spend

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/_types.py                       |   2 +
 litellm/proxy/db/db_spend_update_writer.py    |  45 ++++++-
 .../redis_update_buffer.py                    |   9 ++
 .../spend_update_queue.py                     |   4 +
 .../test_redis_update_buffer.py               |  45 +++++++
 .../proxy/db/test_db_spend_update_writer.py   | 116 ++++++++++++++++++
 6 files changed, 219 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 9f85c3d1894..1d3992ab800 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -246,6 +246,7 @@ class Litellm_EntityType(enum.Enum):
     TEAM = "team"
     TEAM_MEMBER = "team_member"
     ORGANIZATION = "organization"
+    ORGANIZATION_MEMBER = "organization_member"
     PROJECT = "project"
     TAG = "tag"
     AGENT = "agent"
@@ -5236,6 +5237,7 @@ class DBSpendUpdateTransactions(TypedDict):
     team_list_transactions: dict[str, float] | None
     team_member_list_transactions: dict[str, float] | None
     org_list_transactions: dict[str, float] | None
+    org_member_list_transactions: ReadOnly[dict[str, float] | None]
     tag_list_transactions: dict[str, float] | None
     agent_list_transactions: dict[str, float] | None
     model_access_group_list_transactions: ReadOnly[dict[str, float] | None]
diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index eaa03c5d7f7..d207ba2f2c6 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -16,6 +16,7 @@ from collections.abc import Mapping, Sequence
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, overload
+from urllib.parse import quote, unquote
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -85,6 +86,10 @@ else:
 RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value})
 
 
+def _org_member_transaction_key(org_id: str, user_id: str) -> str:
+    return f"organization_id::{quote(org_id, safe='')}::user_id::{quote(user_id, safe='')}"
+
+
 def _is_batch_cost_row(payload: SpendLogsPayload) -> bool:
     return payload.get("call_type") == CallTypes.aretrieve_batch.value and payload.get("status") == "success"
 
@@ -110,6 +115,7 @@ class _SpendBatch(Protocol):
     litellm_teamtable: BatchTable
     litellm_teammembership: BatchTable
     litellm_organizationtable: BatchTable
+    litellm_organizationmembership: BatchTable
     litellm_tagtable: BatchTable
     litellm_agentstable: BatchTable
     litellm_modelaccessgroupbudgettable: BatchTable
@@ -666,6 +672,7 @@ class DBSpendUpdateWriter:
             await self._update_org_db(
                 response_cost=response_cost,
                 org_id=org_id,
+                user_id=user_id,
                 prisma_client=prisma_client,
             )
         except Exception:
@@ -900,6 +907,7 @@ class DBSpendUpdateWriter:
         self,
         response_cost: float | None,
         org_id: str | None,
+        user_id: str | None,
         prisma_client: PrismaClient | None,
     ):
         try:
@@ -916,6 +924,15 @@ class DBSpendUpdateWriter:
                     response_cost=response_cost,
                 )
             )
+
+            if user_id is not None:
+                await self.spend_update_queue.add_update(
+                    update=SpendUpdateQueueItem(
+                        entity_type=Litellm_EntityType.ORGANIZATION_MEMBER,
+                        entity_id=_org_member_transaction_key(org_id, user_id),
+                        response_cost=response_cost,
+                    )
+                )
         except Exception as e:
             spend_log_error(
                 "Spend tracking - failed to enqueue org spend update. org_id=%s, response_cost=%s - %s",
@@ -1163,14 +1180,15 @@ class DBSpendUpdateWriter:
                 if db_spend_update_transactions is not None:
                     verbose_proxy_logger.info(
                         "Spend tracking - committing spend updates from Redis to DB: "
-                        "keys=%d, users=%d, teams=%d, orgs=%d, end_users=%d, team_members=%d, tags=%d, agents=%d, "
-                        "model_access_groups=%d",
+                        "keys=%d, users=%d, teams=%d, orgs=%d, end_users=%d, team_members=%d, org_members=%d, tags=%d, "
+                        "agents=%d, model_access_groups=%d",
                         len(db_spend_update_transactions.get("key_list_transactions") or {}),
                         len(db_spend_update_transactions.get("user_list_transactions") or {}),
                         len(db_spend_update_transactions.get("team_list_transactions") or {}),
                         len(db_spend_update_transactions.get("org_list_transactions") or {}),
                         len(db_spend_update_transactions.get("end_user_list_transactions") or {}),
                         len(db_spend_update_transactions.get("team_member_list_transactions") or {}),
+                        len(db_spend_update_transactions.get("org_member_list_transactions") or {}),
                         len(db_spend_update_transactions.get("tag_list_transactions") or {}),
                         len(db_spend_update_transactions.get("agent_list_transactions") or {}),
                         len(db_spend_update_transactions.get("model_access_group_list_transactions") or {}),
@@ -1708,6 +1726,29 @@ class DBSpendUpdateWriter:
                         proxy_logging_obj=proxy_logging_obj,
                     )
 
+        org_member_list_transactions: Final = db_spend_update_transactions.get("org_member_list_transactions")
+        verbose_proxy_logger.debug("Org Membership Spend transactions: %s", org_member_list_transactions)
+        if org_member_list_transactions is not None and len(org_member_list_transactions.keys()) > 0:
+            for i in range(n_retry_times + 1):
+                start_time = time.time()
+                try:
+                    async with _spend_update_tx(prisma_client) as transaction, transaction.batch_() as batcher:
+                        for key, response_cost in sorted(org_member_list_transactions.items()):
+                            _, quoted_org_id, _, quoted_user_id = key.split("::")
+                            batcher.litellm_organizationmembership.update_many(
+                                where={"organization_id": unquote(quoted_org_id), "user_id": unquote(quoted_user_id)},
+                                data={"spend": {"increment": response_cost}},
+                            )
+                    break
+                except Exception as e:
+                    await self._handle_spend_update_failure(
+                        e=e,
+                        attempt=i,
+                        n_retry_times=n_retry_times,
+                        start_time=start_time,
+                        proxy_logging_obj=proxy_logging_obj,
+                    )
+
         ### UPDATE TAG TABLE ###
         tag_list_transactions: Final = db_spend_update_transactions["tag_list_transactions"]
         await DBSpendUpdateWriter._update_entity_spend_in_db(
diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py
index c06f2e04aca..6f49a00b763 100644
--- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py
+++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py
@@ -69,6 +69,7 @@ _SpendTransactionField: TypeAlias = Literal[
     "team_list_transactions",
     "team_member_list_transactions",
     "org_list_transactions",
+    "org_member_list_transactions",
     "tag_list_transactions",
     "agent_list_transactions",
     "model_access_group_list_transactions",
@@ -81,6 +82,7 @@ _SPEND_TRANSACTION_FIELDS: Final[tuple[_SpendTransactionField, ...]] = (
     "team_list_transactions",
     "team_member_list_transactions",
     "org_list_transactions",
+    "org_member_list_transactions",
     "tag_list_transactions",
     "agent_list_transactions",
     "model_access_group_list_transactions",
@@ -412,6 +414,10 @@ class RedisUpdateBuffer:
                     Litellm_EntityType.ORGANIZATION,
                     db_spend_update_transactions.get("org_list_transactions"),
                 ),
+                (
+                    Litellm_EntityType.ORGANIZATION_MEMBER,
+                    db_spend_update_transactions.get("org_member_list_transactions"),
+                ),
                 (
                     Litellm_EntityType.TAG,
                     db_spend_update_transactions.get("tag_list_transactions"),
@@ -876,6 +882,9 @@ class RedisUpdateBuffer:
                 list_of_transactions, "team_member_list_transactions"
             ),
             org_list_transactions=_merged_entity_transactions(list_of_transactions, "org_list_transactions"),
+            org_member_list_transactions=_merged_entity_transactions(
+                list_of_transactions, "org_member_list_transactions"
+            ),
             tag_list_transactions=_merged_entity_transactions(list_of_transactions, "tag_list_transactions"),
             agent_list_transactions=_merged_entity_transactions(list_of_transactions, "agent_list_transactions"),
             model_access_group_list_transactions=_merged_entity_transactions(
diff --git a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py
index 8c0076b10c1..bc068d10daf 100644
--- a/litellm/proxy/db/db_transaction_queue/spend_update_queue.py
+++ b/litellm/proxy/db/db_transaction_queue/spend_update_queue.py
@@ -137,6 +137,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
             team_list_transactions={},
             team_member_list_transactions={},
             org_list_transactions={},
+            org_member_list_transactions={},
             tag_list_transactions={},
             agent_list_transactions={},
             model_access_group_list_transactions={},
@@ -150,6 +151,7 @@ class SpendUpdateQueue(BaseUpdateQueue):
             Litellm_EntityType.TEAM: "team_list_transactions",
             Litellm_EntityType.TEAM_MEMBER: "team_member_list_transactions",
             Litellm_EntityType.ORGANIZATION: "org_list_transactions",
+            Litellm_EntityType.ORGANIZATION_MEMBER: "org_member_list_transactions",
             Litellm_EntityType.TAG: "tag_list_transactions",
             Litellm_EntityType.AGENT: "agent_list_transactions",
             Litellm_EntityType.MODEL_ACCESS_GROUP: "model_access_group_list_transactions",
@@ -188,6 +190,8 @@ class SpendUpdateQueue(BaseUpdateQueue):
                 transactions_dict = db_spend_update_transactions["team_member_list_transactions"]
             elif dict_key == "org_list_transactions":
                 transactions_dict = db_spend_update_transactions["org_list_transactions"]
+            elif dict_key == "org_member_list_transactions":
+                transactions_dict = db_spend_update_transactions["org_member_list_transactions"]
             elif dict_key == "tag_list_transactions":
                 transactions_dict = db_spend_update_transactions["tag_list_transactions"]
             elif dict_key == "agent_list_transactions":
diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
index 8f3508fc4e9..817c86a1bdf 100644
--- a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
+++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
@@ -266,6 +266,51 @@ async def test_get_all_transactions_from_redis_buffer_pipeline(redis_update_buff
     assert popped_keys[6] == REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY
 
 
+@pytest.mark.asyncio
+async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_failure(
+    redis_update_buffer, mock_redis_cache
+):
+    from litellm.proxy._types import Litellm_EntityType
+    from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
+        DailySpendUpdateQueue,
+    )
+    from litellm.proxy.db.db_transaction_queue.spend_update_queue import (
+        SpendUpdateQueue,
+    )
+
+    member_key = "organization_id::org-1::user_id::user-1"
+    pod_json = json.dumps({"org_member_list_transactions": {member_key: 0.25}})
+    mock_redis_cache.async_lpop_pipeline = AsyncMock(
+        return_value=[[pod_json, pod_json], None, None, None, None, None, None]
+    )
+
+    (db_spend, *_rest) = await redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline()
+
+    assert db_spend is not None
+    assert db_spend["org_member_list_transactions"] == {member_key: 0.5}
+
+    mock_redis_cache.async_rpush_pipeline = AsyncMock(side_effect=ConnectionError("redis went away"))
+    spend_queue = SpendUpdateQueue()
+    await spend_queue.add_update(
+        {
+            "entity_type": Litellm_EntityType.ORGANIZATION_MEMBER,
+            "entity_id": member_key,
+            "response_cost": 1.5,
+        }
+    )
+    await redis_update_buffer.store_in_memory_spend_updates_in_redis(
+        spend_update_queue=spend_queue,
+        daily_spend_update_queue=DailySpendUpdateQueue(),
+        daily_team_spend_update_queue=DailySpendUpdateQueue(),
+        daily_org_spend_update_queue=DailySpendUpdateQueue(),
+        daily_end_user_spend_update_queue=DailySpendUpdateQueue(),
+        daily_agent_spend_update_queue=DailySpendUpdateQueue(),
+    )
+
+    restored_spend = await spend_queue.flush_and_get_aggregated_db_spend_update_transactions()
+    assert restored_spend["org_member_list_transactions"] == {member_key: 1.5}
+
+
 @pytest.mark.asyncio
 async def test_get_all_transactions_from_redis_buffer_pipeline_no_redis():
     """When redis_cache is None, should return all Nones"""
diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
index 5e977712a1e..64001146e05 100644
--- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
+++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
@@ -944,6 +944,121 @@ async def test_commit_spend_updates_to_db_increments_team_member_spend_and_total
     }
 
 
+@pytest.mark.asyncio
+async def test_org_spend_increments_organization_membership_row_for_the_calling_user():
+    """A request made with a user_id inside an org must increment that user's
+    LiteLLM_OrganizationMembership.spend, not only the org total, or the
+    Organizations > Members UI renders '-' for every member."""
+    db_writer = DBSpendUpdateWriter()
+    await db_writer._update_org_db(
+        response_cost=0.75,
+        org_id="org-abc",
+        user_id="user-xyz",
+        prisma_client=MagicMock(),
+    )
+    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+
+    mock_batcher = MagicMock()
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.tx = MagicMock(return_value=_good_tx(mock_batcher))
+    proxy_logging = MagicMock()
+    proxy_logging.call_details = {}
+
+    await db_writer._commit_spend_updates_to_db(
+        prisma_client=mock_prisma_client,
+        n_retry_times=0,
+        proxy_logging_obj=proxy_logging,
+        db_spend_update_transactions=transactions,
+    )
+
+    mock_batcher.litellm_organizationtable.update_many.assert_called_once_with(
+        where={"organization_id": "org-abc"},
+        data={"spend": {"increment": 0.75}},
+    )
+    mock_batcher.litellm_organizationmembership.update_many.assert_called_once_with(
+        where={"organization_id": "org-abc", "user_id": "user-xyz"},
+        data={"spend": {"increment": 0.75}},
+    )
+
+
+@pytest.mark.asyncio
+async def test_org_spend_without_user_id_leaves_organization_membership_untouched():
+    db_writer = DBSpendUpdateWriter()
+    await db_writer._update_org_db(
+        response_cost=0.75,
+        org_id="org-abc",
+        user_id=None,
+        prisma_client=MagicMock(),
+    )
+    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+
+    mock_batcher = MagicMock()
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.tx = MagicMock(return_value=_good_tx(mock_batcher))
+    proxy_logging = MagicMock()
+    proxy_logging.call_details = {}
+
+    await db_writer._commit_spend_updates_to_db(
+        prisma_client=mock_prisma_client,
+        n_retry_times=0,
+        proxy_logging_obj=proxy_logging,
+        db_spend_update_transactions=transactions,
+    )
+
+    mock_batcher.litellm_organizationtable.update_many.assert_called_once()
+    mock_batcher.litellm_organizationmembership.update_many.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_org_spend_keeps_member_attribution_when_ids_contain_the_key_delimiter():
+    db_writer = DBSpendUpdateWriter()
+    await db_writer._update_org_db(
+        response_cost=0.75,
+        org_id="division::west",
+        user_id="user::42",
+        prisma_client=MagicMock(),
+    )
+    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+
+    mock_batcher = MagicMock()
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.tx = MagicMock(return_value=_good_tx(mock_batcher))
+    proxy_logging = MagicMock()
+    proxy_logging.call_details = {}
+
+    await db_writer._commit_spend_updates_to_db(
+        prisma_client=mock_prisma_client,
+        n_retry_times=0,
+        proxy_logging_obj=proxy_logging,
+        db_spend_update_transactions=transactions,
+    )
+
+    mock_batcher.litellm_organizationmembership.update_many.assert_called_once_with(
+        where={"organization_id": "division::west", "user_id": "user::42"},
+        data={"spend": {"increment": 0.75}},
+    )
+
+
+@pytest.mark.asyncio
+async def test_batch_database_updates_queues_org_member_spend_for_the_request_user():
+    db_writer = DBSpendUpdateWriter()
+    await db_writer._batch_database_updates(
+        response_cost=0.1,
+        user_id="u1",
+        hashed_token="t1",
+        team_id=None,
+        org_id="org1",
+        end_user_id=None,
+        prisma_client=MagicMock(),
+        litellm_proxy_budget_name=None,
+        payload={"request_id": "req-1", "model": "gpt-4o-mini", "spend": 0.1},
+    )
+    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+
+    assert transactions["org_list_transactions"] == {"org1": 0.1}
+    assert transactions["org_member_list_transactions"] == {"organization_id::org1::user_id::u1": 0.1}
+
+
 @pytest.mark.asyncio
 async def test_add_spend_log_transaction_to_daily_tag_transaction_with_request_id():
     """
@@ -2904,6 +3019,7 @@ async def test_update_daily_spend_retries_deadlock(monkeypatch):
         ("team_list_transactions", "team-1"),
         ("team_member_list_transactions", "team_id::team-1::user_id::user-1"),
         ("org_list_transactions", "org-1"),
+        ("org_member_list_transactions", "organization_id::org-1::user_id::user-1"),
         ("tag_list_transactions", "tag-1"),
         ("agent_list_transactions", "agent-1"),
     ],

From 864f4a7a0e70ad19b4c251e1a2447ea1aa7965bf Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Tue, 15 Sep 2026 12:29:19 -0700
Subject: [PATCH 18/67] feat(auto-router): add per-model Fast mode toggle

---
 litellm/llms/anthropic/common_utils.py        |   7 +
 ...odel_prices_and_context_window_backup.json |   2 +
 litellm/router.py                             |   5 +
 litellm/types/router.py                       |   1 +
 model_prices_and_context_window.json          |   2 +
 model_prices_and_context_window.schema.json   |   3 +
 tests/test_litellm/test_router.py             |  77 ++++++++++
 tests/test_litellm/test_utils.py              |   1 +
 .../add_model/ComplexityRouterConfig.tsx      |  19 ++-
 ...plexityRouterFastMode.integration.test.tsx | 143 ++++++++++++++++++
 .../add_model/TierModelEffortRows.tsx         | 118 +++++++++------
 .../build_complexity_router_config.test.ts    |  13 ++
 .../add_model/complexity_router_tiers.test.ts |  23 +++
 .../add_model/complexity_router_tiers.ts      |  17 ++-
 .../llm_calls/fetch_models.test.tsx           |  17 +++
 .../src/components/llm_calls/fetch_models.tsx |   3 +
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   5 +
 17 files changed, 400 insertions(+), 56 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx

diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py
index 87c4ec8938e..d35a9372058 100644
--- a/litellm/llms/anthropic/common_utils.py
+++ b/litellm/llms/anthropic/common_utils.py
@@ -539,6 +539,13 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         value: Final = litellm.model_cost.get(model, {}).get(key)
         return value if isinstance(value, bool) else None
 
+    @staticmethod
+    def supports_fast_mode(model: str, custom_llm_provider: str) -> bool:
+        return (
+            custom_llm_provider == "anthropic"
+            and AnthropicModelInfo._get_exact_model_capability(model, "supports_fast_mode") is True
+        )
+
     @staticmethod
     def _get_provider_resolved_capability(model: str, key: str, custom_llm_provider: str) -> bool | None:
         """Resolve boolean capability ``key`` for ``model`` under the caller's provider.
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 9f91cf82f41..19c438ef52d 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -14062,6 +14062,7 @@
         },
         "supports_output_config": true,
         "supports_speed": true,
+        "supports_fast_mode": true,
         "prompt_cache_min_tokens": 512,
         "source": "https://platform.claude.com/docs/en/about-claude/pricing"
     },
@@ -14103,6 +14104,7 @@
         },
         "supports_output_config": true,
         "supports_speed": true,
+        "supports_fast_mode": true,
         "prompt_cache_min_tokens": 1024,
         "source": "https://platform.claude.com/docs/en/about-claude/pricing"
     },
diff --git a/litellm/router.py b/litellm/router.py
index fcdcf91c2cf..34e7a78f17b 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -99,6 +99,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
     mask_sensitive_structure,
 )
 from litellm.litellm_core_utils.token_counter import offload_token_count
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.base_llm.passthrough.transformation import replace_path_segment
 from litellm.llms.base_llm.vector_store.transformation import (
     RouterVectorStoreEmbeddingExecutor,
@@ -10794,6 +10795,7 @@ class Router:
                         "model_group": user_facing_model_group_name,
                         "providers": [llm_provider],
                         **model_info,
+                        "supports_fast_mode": True,
                         "supported_reasoning_efforts": None,
                     }
                 )
@@ -10872,6 +10874,9 @@ class Router:
                 if model_info.get("rpm", None) is not None and _deployment_rpm is None:
                     _deployment_rpm = model_info.get("rpm")
 
+            model_group_info.supports_fast_mode = model_group_info.supports_fast_mode and (
+                AnthropicModelInfo.supports_fast_mode(litellm_model, llm_provider)
+            )
             deployment_reasoning_efforts = (
                 resolve_supported_reasoning_efforts(  # rebind-ok: recalculated per deployment
                     model_info, deployment_is_mapped=deployment_is_mapped
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 7732413b593..7c3e4d6943f 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -722,6 +722,7 @@ class ModelGroupInfo(BaseModel):
     supports_url_context: bool = Field(default=False)
     supports_reasoning: bool = Field(default=False)
     supports_function_calling: bool = Field(default=False)
+    supports_fast_mode: bool = Field(default=False)
     supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None)
     supported_openai_params: list[str] | None = Field(default=[])
     configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 9f91cf82f41..19c438ef52d 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -14062,6 +14062,7 @@
         },
         "supports_output_config": true,
         "supports_speed": true,
+        "supports_fast_mode": true,
         "prompt_cache_min_tokens": 512,
         "source": "https://platform.claude.com/docs/en/about-claude/pricing"
     },
@@ -14103,6 +14104,7 @@
         },
         "supports_output_config": true,
         "supports_speed": true,
+        "supports_fast_mode": true,
         "prompt_cache_min_tokens": 1024,
         "source": "https://platform.claude.com/docs/en/about-claude/pricing"
     },
diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json
index c2490041cf7..130cc6873fa 100644
--- a/model_prices_and_context_window.schema.json
+++ b/model_prices_and_context_window.schema.json
@@ -716,6 +716,9 @@
         "supports_embedding_image_input": {
           "type": "boolean"
         },
+        "supports_fast_mode": {
+          "type": "boolean"
+        },
         "supports_forced_tool_use": {
           "type": "boolean"
         },
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 9c29b9d829a..ea90c167845 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -12056,6 +12056,83 @@ def test_model_group_info_reasoning_efforts_are_unknown_when_any_deployment_is_o
 
 
 
+@pytest.mark.parametrize(
+    "model,provider,expected",
+    [
+        ("anthropic/claude-opus-5", None, True),
+        ("claude-opus-4-8", None, True),
+        ("anthropic/claude-opus-4-7", None, False),
+        ("anthropic/claude-opus-4-6", None, False),
+        ("anthropic/claude-sonnet-5", None, False),
+        ("anthropic/off-map-opus", None, False),
+        ("vertex_ai/claude-opus-5", None, False),
+        ("bedrock/claude-opus-5", None, False),
+        ("claude-opus-5", "vertex_ai", False),
+        ("claude-opus-5", "bedrock", False),
+    ],
+)
+@pytest.mark.parametrize("operator_flag", [True, False])
+def test_model_group_info_fast_mode_uses_exact_provider_catalog(
+    local_model_cost_map: None, model: str, provider: str | None, expected: bool, operator_flag: bool
+) -> None:
+    router: Final = Router(model_list=[{
+        "model_name": "fast-group",
+        "litellm_params": {"model": model, "custom_llm_provider": provider, "api_key": "fake-key"},
+        "model_info": {"supports_fast_mode": operator_flag},
+    }])
+
+    result: Final = router.get_model_group_info("fast-group")
+
+    assert result is not None
+    assert result.supports_fast_mode is expected
+
+
+@pytest.mark.parametrize("flag", [None, False, "true", 1])
+def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean(
+    local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, flag: object
+) -> None:
+    entry: Final = {key: value for key, value in litellm.model_cost["claude-opus-5"].items()
+                   if key != "supports_fast_mode"}
+    if flag is not None:
+        entry["supports_fast_mode"] = flag
+    monkeypatch.setitem(litellm.model_cost, "claude-opus-5", entry)
+    router: Final = Router(model_list=[{
+        "model_name": "fast-group",
+        "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "fake-key"},
+        "model_info": {"supports_fast_mode": True},
+    }])
+
+    result: Final = router.get_model_group_info("fast-group")
+
+    assert result is not None
+    assert result.supports_fast_mode is False
+
+
+@pytest.mark.parametrize("other_model,expected", [
+    ("anthropic/claude-opus-4-8", True),
+    ("anthropic/claude-opus-4-7", False),
+    ("anthropic/off-map-opus", False),
+    ("vertex_ai/claude-opus-5", False),
+    ("bedrock/claude-opus-5", False),
+])
+@pytest.mark.parametrize("reverse", [True, False])
+def test_model_group_info_fast_mode_requires_every_deployment(
+    local_model_cost_map: None, other_model: str, expected: bool, reverse: bool
+) -> None:
+    models: Final = (other_model, "anthropic/claude-opus-5") if reverse else (
+        "anthropic/claude-opus-5", other_model
+    )
+    router: Final = Router(model_list=[{
+        "model_name": "fast-group",
+        "litellm_params": {"model": model, "api_key": "fake-key"},
+    } for model in models])
+
+    result: Final = router.get_model_group_info("fast-group")
+
+    assert result is not None
+    assert result.supports_fast_mode is expected
+
+
 def test_model_group_info_surfaces_supports_parallel_function_calling(local_model_cost_map):
     """``/model_group/info`` folds each deployment's registry flags into the group; a deployment whose
     registry entry declares parallel function calling must flip the group to True instead of False."""
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 8bf8489fc52..6e6acd5aabf 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -1165,6 +1165,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                 "supports_sampling_params": {"type": "boolean"},
                 "supports_output_config": {"type": "boolean"},
                 "supports_speed": {"type": "boolean"},
+                "supports_fast_mode": {"type": "boolean"},
                 "supported_audio_formats": {
                     "type": "array",
                     "items": {
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index febcde269f7..64b17751e95 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -42,9 +42,10 @@ import { Restricted, restrictedBy } from "./TierRestrictions";
 import { type TierSetAction, applyTierSetAction, setFallbackTier } from "./tier_set_actions";
 import {
   ReasoningEffort,
+  TierModelParamChange,
   TierModelParamsByTier,
   classifierEffortOptionsForModels,
-  setTierModelReasoningEffort,
+  setTierModelParam,
   tierEffortOptionsForModels,
   tierRowLabel,
 } from "./complexity_router_tiers";
@@ -613,6 +614,9 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
   const exitToBuiltInTiers = () => dispatch({ kind: "restore" });
 
   const tierEffortOptionsByModel = tierEffortOptionsForModels(modelInfo);
+  const fastModeByModel = Object.fromEntries(
+    modelInfo.map((model) => [model.model_group, model.supports_fast_mode === true]),
+  );
   const classifierEffortOptionsByModel = classifierEffortOptionsForModels(modelInfo);
 
   // Embedding models can't serve a chat-completion role, so they're excluded here.
@@ -623,12 +627,11 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
       label: model.model_group,
     }));
 
-  const handleTierModelEffortChange = (tier: string, model: string, effort: ReasoningEffort | undefined) => {
+  const handleTierModelParamChange = (tier: string, model: string, change: TierModelParamChange) =>
     onChange({
       ...value,
-      tier_model_params: setTierModelReasoningEffort(value.tier_model_params, tier, model, effort),
+      tier_model_params: setTierModelParam(value.tier_model_params, tier, model, change),
     });
-  };
 
   // Clearing the select drops the key entirely rather than storing "", so an emptied pin reads as
   // "track the tiers" everywhere downstream instead of as a blank model name.
@@ -726,7 +729,13 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
                     models={row.models}
                     effortOptionsByModel={tierEffortOptionsByModel}
                     paramsByModel={row.params}
-                    onEffortChange={(model, effort) => handleTierModelEffortChange(row.id, model, effort)}
+                    fastModeByModel={fastModeByModel}
+                    onEffortChange={(model, effort) =>
+                      handleTierModelParamChange(row.id, model, ["reasoning_effort", effort])
+                    }
+                    onFastModeChange={(model, enabled) =>
+                      handleTierModelParamChange(row.id, model, ["speed", enabled ? "fast" : undefined])
+                    }
                   />
                   {row.models.length > 1 && (
                     <span className="text-xs text-muted-foreground">
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx
new file mode 100644
index 00000000000..34d14091bde
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterFastMode.integration.test.tsx
@@ -0,0 +1,143 @@
+import userEvent from "@testing-library/user-event";
+import React from "react";
+import { describe, expect, it, vi } from "vitest";
+import { renderWithProviders, screen } from "../../../tests/test-utils";
+import {
+  buildUpdatedComplexityRouterConfig,
+  hydrateComplexityRouterConfig,
+} from "../edit_auto_router/edit_auto_router_modal";
+import type { ModelGroup } from "../llm_calls/fetch_models";
+import ComplexityRouterConfig, { type ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+
+const modelInfo: ModelGroup[] = [
+  { model_group: "primary", supported_reasoning_efforts: ["low", "high"], supports_fast_mode: true },
+  { model_group: "secondary", supports_fast_mode: true },
+  { model_group: "blocked", supported_reasoning_efforts: ["low"], supports_fast_mode: false },
+  { model_group: "missing", supported_reasoning_efforts: ["low"] },
+];
+
+it.each([false, true])("edits and round-trips independent model settings with custom tiers=%s", async (custom) => {
+  const user = userEvent.setup();
+  const tier = custom ? "custom-a" : "COMPLEX";
+  const otherTier = custom ? "custom-b" : "REASONING";
+  const label = custom ? "Interactive" : "Complex";
+  const models = ["primary", "secondary", "blocked", "missing"];
+  const initial: ComplexityRouterConfigValue = {
+    tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: models, REASONING: ["primary"] },
+    classifier_type: "heuristic",
+    ...(custom && {
+      custom_tier_set: {
+        tiers: [
+          { id: tier, name: label, definition: "Interactive requests", models },
+          { id: otherTier, name: "Deliberate", definition: "Careful requests", models: ["primary"] },
+        ],
+        fallback_tier_id: tier,
+      },
+    }),
+    tier_model_params: {
+      [tier]: {
+        primary: { reasoning_effort: "high", max_tokens: 1024 },
+        secondary: { speed: "fast" },
+        blocked: { speed: "fast" },
+      },
+      [otherTier]: { primary: { speed: "fast", reasoning_effort: "low" } },
+    },
+  };
+  const onChange = vi.fn<(value: ComplexityRouterConfigValue) => void>();
+  const editor = (value: ComplexityRouterConfigValue) => (
+    <ComplexityRouterConfig modelInfo={modelInfo} value={value} onChange={onChange} />
+  );
+  const view = renderWithProviders(editor(initial));
+  const fast = () => screen.getByRole("switch", { name: `Fast mode for primary in the ${label} tier` });
+
+  expect(screen.getAllByRole("switch", { name: /^Fast mode for/ })).toHaveLength(3);
+  expect(screen.queryByRole("switch", { name: /^Fast mode for (blocked|missing)/ })).not.toBeInTheDocument();
+  expect(screen.queryByRole("combobox", { name: /^Reasoning effort for secondary/ })).not.toBeInTheDocument();
+  expect(screen.getByRole("switch", { name: `Fast mode for secondary in the ${label} tier` })).toBeChecked();
+  expect(fast()).not.toBeChecked();
+  expect(onChange).not.toHaveBeenCalled();
+
+  await user.click(fast());
+  const enabled = onChange.mock.lastCall![0];
+  expect(enabled.tier_model_params).toEqual({
+    ...initial.tier_model_params,
+    [tier]: {
+      ...initial.tier_model_params![tier],
+      primary: { reasoning_effort: "high", max_tokens: 1024, speed: "fast" },
+    },
+  });
+  const saved = buildUpdatedComplexityRouterConfig({}, enabled);
+  expect(saved.tier_model_configs).toEqual({
+    [custom ? label : tier]: [
+      { model_name: "primary", litellm_params: { reasoning_effort: "high", max_tokens: 1024, speed: "fast" } },
+      { model_name: "secondary", litellm_params: { speed: "fast" } },
+      { model_name: "blocked", litellm_params: { speed: "fast" } },
+    ],
+    [custom ? "Deliberate" : otherTier]: [
+      { model_name: "primary", litellm_params: { speed: "fast", reasoning_effort: "low" } },
+    ],
+  });
+  const reopened = hydrateComplexityRouterConfig(saved, undefined);
+  const reopenedTier = custom ? reopened.custom_tier_set!.tiers[0].id : tier;
+  view.rerender(editor(reopened));
+  expect(fast()).toBeChecked();
+
+  await user.click(screen.getByRole("combobox", { name: `Reasoning effort for primary in the ${label} tier` }));
+  await user.click(await screen.findByRole("option", { name: "low" }));
+  const effortChanged = onChange.mock.lastCall![0];
+  expect(effortChanged.tier_model_params?.[reopenedTier].primary).toEqual({
+    reasoning_effort: "low",
+    max_tokens: 1024,
+    speed: "fast",
+  });
+  view.rerender(editor(effortChanged));
+  await user.click(fast());
+  const disabled = onChange.mock.lastCall![0];
+  expect(disabled.tier_model_params).toEqual({
+    ...effortChanged.tier_model_params,
+    [reopenedTier]: {
+      ...effortChanged.tier_model_params![reopenedTier],
+      primary: { reasoning_effort: "low", max_tokens: 1024 },
+    },
+  });
+  view.rerender(editor(disabled));
+  expect(fast()).not.toBeChecked();
+
+  const picker = () => screen.getByRole("combobox", { name: `Select model(s) for ${label.toLowerCase()} queries` });
+  await user.click(picker());
+  await user.click(await screen.findByRole("option", { name: "primary" }));
+  await user.keyboard("{Escape}");
+  const deselected = onChange.mock.lastCall![0];
+  expect(deselected.tier_model_params?.[reopenedTier]).toEqual({
+    secondary: { speed: "fast" },
+    blocked: { speed: "fast" },
+  });
+  view.rerender(editor(deselected));
+  expect(screen.queryByRole("switch", { name: `Fast mode for primary in the ${label} tier` })).not.toBeInTheDocument();
+  await user.click(picker());
+  await user.click(await screen.findByRole("option", { name: "primary" }));
+  await user.keyboard("{Escape}");
+  const reselected = onChange.mock.lastCall![0];
+  view.rerender(editor(reselected));
+  expect(fast()).not.toBeChecked();
+  expect(screen.getByRole("combobox", { name: `Reasoning effort for primary in the ${label} tier` })).toHaveTextContent(
+    "Default",
+  );
+});
+
+describe("Fast mode metadata", () => {
+  it("offers nothing before model capabilities load and leaves stored speed untouched", () => {
+    const value: ComplexityRouterConfigValue = {
+      tiers: { SIMPLE: ["primary"], MEDIUM: [], COMPLEX: [], REASONING: [] },
+      classifier_type: "heuristic",
+      tier_model_params: { SIMPLE: { primary: { speed: "fast" } } },
+    };
+    const onChange = vi.fn();
+    renderWithProviders(<ComplexityRouterConfig modelInfo={[]} value={value} onChange={onChange} />);
+    expect(screen.queryByRole("switch", { name: /^Fast mode for/ })).not.toBeInTheDocument();
+    expect(onChange).not.toHaveBeenCalled();
+    expect(buildUpdatedComplexityRouterConfig({}, value).tier_model_configs).toEqual({
+      SIMPLE: [{ model_name: "primary", litellm_params: { speed: "fast" } }],
+    });
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
index ec9705b9451..54afa28fb6e 100644
--- a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
@@ -1,5 +1,6 @@
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { SimpleTooltip } from "@/components/ui/tooltip";
+import { Switch } from "@/components/ui/switch";
 import { Info } from "lucide-react";
 import React from "react";
 import { ReasoningEffort, TierModelParams } from "./complexity_router_tiers";
@@ -18,6 +19,8 @@ interface TierModelEffortRowsProps {
   effortOptionsByModel: Record<string, string[]>;
   paramsByModel: Record<string, TierModelParams> | undefined;
   onEffortChange: (model: string, effort: ReasoningEffort | undefined) => void;
+  fastModeByModel?: Record<string, boolean>;
+  onFastModeChange: (model: string, enabled: boolean) => void;
 }
 
 export interface TierEffortRow {
@@ -29,13 +32,16 @@ export interface TierEffortRow {
 /**
  * A stored effort outside the model's supported set (hand-authored, or capabilities changed since
  * it was saved) is listed anyway, so the row renders with its value selected and can be cleared.
- * Only a model with no supported level and nothing stored drops out.
  */
 export const tierEffortRows = ({
   models,
   effortOptionsByModel,
   paramsByModel,
-}: Pick<TierModelEffortRowsProps, "models" | "effortOptionsByModel" | "paramsByModel">): TierEffortRow[] =>
+  fastModeByModel,
+}: Pick<
+  TierModelEffortRowsProps,
+  "models" | "effortOptionsByModel" | "paramsByModel" | "fastModeByModel"
+>): TierEffortRow[] =>
   models
     .map((model) => {
       const effort = storedEffort(paramsByModel?.[model]);
@@ -43,56 +49,74 @@ export const tierEffortRows = ({
       const listed = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported;
       return { model, effort, options: Array.from(new Set(listed)) };
     })
-    .filter(({ options }) => options.length > 0);
+    .filter(({ model, options }) => options.length > 0 || fastModeByModel?.[model] === true);
 
-const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = ({
-  tierLabel,
-  models,
-  effortOptionsByModel,
-  paramsByModel,
-  onEffortChange,
-}) => {
-  const rows = tierEffortRows({ models, effortOptionsByModel, paramsByModel });
+const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = (props) => {
+  const { tierLabel, paramsByModel, onEffortChange, fastModeByModel, onFastModeChange } = props;
+  const rows = tierEffortRows(props);
   if (rows.length === 0) return null;
   return (
     <div className="mt-2 space-y-1">
-      <div className="flex items-center gap-1">
-        <span className="text-xs font-medium text-muted-foreground">Reasoning effort</span>
-        <SimpleTooltip
-          content={`Sent as reasoning_effort on requests this tier routes to the model, overriding the caller's value. Default leaves the request untouched.`}
-        >
-          <Info className="size-3 text-muted-foreground/70" />
-        </SimpleTooltip>
-      </div>
-      {rows.map(({ model, effort, options }) => (
-        <div key={model} className="flex items-center justify-between gap-2">
-          <span className="truncate text-xs">{model}</span>
-          <Select
-            items={[
-              { value: PROVIDER_DEFAULT, label: "Default" },
-              ...options.map((option) => ({ value: option, label: option })),
-            ]}
-            value={effort ?? PROVIDER_DEFAULT}
-            onValueChange={(selected: string | null) =>
-              selected !== null && onEffortChange(model, selected === PROVIDER_DEFAULT ? undefined : selected)
-            }
+      {rows.some(({ options }) => options.length > 0) && (
+        <div className="flex items-center gap-1">
+          <span className="text-xs font-medium text-muted-foreground">Reasoning effort</span>
+          <SimpleTooltip
+            content={`Sent as reasoning_effort on requests this tier routes to the model, overriding the caller's value. Default leaves the request untouched.`}
           >
-            <SelectTrigger
-              size="sm"
-              className="w-36"
-              aria-label={`Reasoning effort for ${model} in the ${tierLabel} tier`}
-            >
-              <SelectValue />
-            </SelectTrigger>
-            <SelectContent>
-              <SelectItem value={PROVIDER_DEFAULT}>Default</SelectItem>
-              {options.map((option) => (
-                <SelectItem key={option} value={option}>
-                  {option}
-                </SelectItem>
-              ))}
-            </SelectContent>
-          </Select>
+            <Info className="size-3 text-muted-foreground/70" />
+          </SimpleTooltip>
+        </div>
+      )}
+      {rows.map(({ model, effort, options }) => (
+        <div key={model} className="flex flex-wrap items-center justify-between gap-2">
+          <span className="min-w-0 flex-1 basis-32 truncate text-xs" title={model}>
+            {model}
+          </span>
+          <div className="flex flex-wrap items-center gap-3">
+            {options.length > 0 && (
+              <Select
+                items={[
+                  { value: PROVIDER_DEFAULT, label: "Default" },
+                  ...options.map((option) => ({ value: option, label: option })),
+                ]}
+                value={effort ?? PROVIDER_DEFAULT}
+                onValueChange={(selected: string | null) =>
+                  selected !== null && onEffortChange(model, selected === PROVIDER_DEFAULT ? undefined : selected)
+                }
+              >
+                <SelectTrigger
+                  size="sm"
+                  className="w-36"
+                  aria-label={`Reasoning effort for ${model} in the ${tierLabel} tier`}
+                >
+                  <SelectValue />
+                </SelectTrigger>
+                <SelectContent>
+                  <SelectItem value={PROVIDER_DEFAULT}>Default</SelectItem>
+                  {options.map((option) => (
+                    <SelectItem key={option} value={option}>
+                      {option}
+                    </SelectItem>
+                  ))}
+                </SelectContent>
+              </Select>
+            )}
+            {fastModeByModel?.[model] === true && (
+              <SimpleTooltip content="Fast mode has higher pricing and requires an eligible provider account. Off removes this tier's speed override and inherits the request or provider default">
+                <label
+                  className="flex items-center gap-2 text-xs"
+                  aria-label={`Fast mode for ${model} in the ${tierLabel} tier`}
+                >
+                  <Switch
+                    size="sm"
+                    checked={paramsByModel?.[model]?.speed === "fast"}
+                    onCheckedChange={(enabled) => onFastModeChange(model, enabled)}
+                  />
+                  Fast mode
+                </label>
+              </SimpleTooltip>
+            )}
+          </div>
         </div>
       ))}
     </div>
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
index 9973aec7616..bc30b591ea9 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
@@ -48,6 +48,19 @@ const baseParams: BuildComplexityRouterConfigParams = {
 };
 
 describe("buildComplexityRouterConfig", () => {
+  it("carries Fast and reasoning overrides independently into a new router payload", () => {
+    const params = { speed: "fast", reasoning_effort: "high", max_tokens: 1024 };
+    const config = buildComplexityRouterConfig({
+      ...baseParams,
+      tiers: { ...tiers, COMPLEX: ["primary"], REASONING: ["secondary"] },
+      tierModelParams: { COMPLEX: { primary: params }, REASONING: { secondary: { speed: "fast" } } },
+    });
+    expect(config.tier_model_configs).toEqual({
+      COMPLEX: [{ model_name: "primary", litellm_params: params }],
+      REASONING: [{ model_name: "secondary", litellm_params: { speed: "fast" } }],
+    });
+  });
+
   it("emits tiers, classifier_type, and escalation_keywords when nothing else is configured", () => {
     const config = buildComplexityRouterConfig(baseParams);
     const expected = {
diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts
index b0fab49e80a..98a3fe792bd 100644
--- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.test.ts
@@ -7,6 +7,7 @@ import {
   serializeTierModelConfigs,
   tierRowLabel,
   setTierModelReasoningEffort,
+  setTierModelParam,
 } from "./complexity_router_tiers";
 import { resolveComplexityDefaultModel } from "./tier_rows";
 
@@ -218,6 +219,28 @@ describe("setTierModelReasoningEffort", () => {
   });
 });
 
+describe("setTierModelParam", () => {
+  it.each(["reasoning_effort", "speed"] as const)("clears only %s and preserves the input", (key) => {
+    const params = { reasoning_effort: "high", speed: "fast", max_tokens: 512 };
+    const current = { COMPLEX: { primary: params, secondary: { speed: "fast" } }, REASONING: { primary: params } };
+    const cleared = setTierModelParam(current, "COMPLEX", "primary", [key, undefined]);
+    expect(cleared).toEqual({
+      ...current,
+      COMPLEX: {
+        ...current.COMPLEX,
+        primary: key === "speed" ? { reasoning_effort: "high", max_tokens: 512 } : { speed: "fast", max_tokens: 512 },
+      },
+    });
+    expect(current.COMPLEX.primary).toEqual({ reasoning_effort: "high", speed: "fast", max_tokens: 512 });
+  });
+
+  it("removes empty records when the only override is Fast", () => {
+    const enabled = setTierModelParam(undefined, "COMPLEX", "primary", ["speed", "fast"]);
+    expect(enabled).toEqual({ COMPLEX: { primary: { speed: "fast" } } });
+    expect(setTierModelParam(enabled, "COMPLEX", "primary", ["speed", undefined])).toBeUndefined();
+  });
+});
+
 describe("pruneTierModelParams", () => {
   it("drops params for models deselected from the tier", () => {
     expect(
diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
index 3fec63518e5..916feaf26c0 100644
--- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
+++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
@@ -114,14 +114,16 @@ export const serializeTierModelConfigs = (
   return serialized.length > 0 ? Object.fromEntries(serialized) : undefined;
 };
 
-export const setTierModelReasoningEffort = (
+export type TierModelParamChange = ["reasoning_effort", ReasoningEffort | undefined] | ["speed", "fast" | undefined];
+
+export const setTierModelParam = (
   current: TierModelParamsByTier | undefined,
   tier: string,
   model: string,
-  effort: ReasoningEffort | undefined,
+  [key, value]: TierModelParamChange,
 ): TierModelParamsByTier | undefined => {
-  const { reasoning_effort: _dropped, ...rest } = current?.[tier]?.[model] ?? {};
-  const params = effort === undefined ? rest : { ...rest, reasoning_effort: effort };
+  const { [key]: _dropped, ...rest } = current?.[tier]?.[model] ?? {};
+  const params = value === undefined ? rest : { ...rest, [key]: value };
   const byModel = Object.fromEntries(
     Object.entries({ ...current?.[tier], [model]: params }).filter(([, value]) => Object.keys(value).length > 0),
   );
@@ -131,6 +133,13 @@ export const setTierModelReasoningEffort = (
   return Object.keys(next).length > 0 ? next : undefined;
 };
 
+export const setTierModelReasoningEffort = (
+  current: TierModelParamsByTier | undefined,
+  tier: string,
+  model: string,
+  effort: ReasoningEffort | undefined,
+): TierModelParamsByTier | undefined => setTierModelParam(current, tier, model, ["reasoning_effort", effort]);
+
 export const pruneTierModelParams = (
   current: TierModelParamsByTier | undefined,
   tier: string,
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
index a3a6c77930b..c65ece4ca1c 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
@@ -52,6 +52,23 @@ describe("fetchAvailableModels", () => {
     ]);
   });
 
+  it("carries only explicitly supported Fast capabilities, not accepted speed parameters", async () => {
+    modelHubCallMock.mockResolvedValue({
+      data: [
+        { model_group: "fast", supports_fast_mode: true },
+        { model_group: "blocked", supports_fast_mode: false },
+        { model_group: "missing", supports_speed: true },
+        { model_group: "unknown", supports_fast_mode: null },
+      ],
+    });
+    expect(await fetchAvailableModels("token")).toEqual([
+      { model_group: "blocked" },
+      { model_group: "fast", supports_fast_mode: true },
+      { model_group: "missing" },
+      { model_group: "unknown" },
+    ]);
+  });
+
   it("preserves absent, unknown, empty, and explicit effort capability states", async () => {
     modelHubCallMock.mockResolvedValue({
       data: [
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
index 1d21b5e43ba..b3df5c9bf65 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
@@ -7,6 +7,7 @@ export interface ModelGroup {
   model_group: string;
   mode?: string;
   supports_reasoning?: boolean;
+  supports_fast_mode?: boolean;
   supported_reasoning_efforts?: string[] | null;
 }
 
@@ -16,6 +17,7 @@ interface AvailableModel {
   id?: string | null;
   mode?: string | null;
   supports_reasoning?: boolean | null;
+  supports_fast_mode?: boolean | null;
   supported_reasoning_efforts?: string[] | null;
 }
 
@@ -25,6 +27,7 @@ const toModelGroup = (item: AvailableModel): ModelGroup => {
     model_group: groupName,
     ...(item.mode && { mode: item.mode }),
     ...(item.supports_reasoning === true && { supports_reasoning: true }),
+    ...(item.supports_fast_mode === true && { supports_fast_mode: true }),
     ...(item.supported_reasoning_efforts !== undefined && {
       supported_reasoning_efforts: item.supported_reasoning_efforts,
     }),
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 371fbbe67bd..a8801f643f4 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -32565,6 +32565,11 @@ export interface components {
             supported_openai_params: string[] | null;
             /** Supported Reasoning Efforts */
             supported_reasoning_efforts?: string[] | null;
+            /**
+             * Supports Fast Mode
+             * @default false
+             */
+            supports_fast_mode: boolean;
             /**
              * Supports Function Calling
              * @default false

From c9ccda210ebc4e55b999f8e4d253e5b3161d574f Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 19:49:58 +0000
Subject: [PATCH 19/67] test(proxy): declare org member spend test bindings
 Final

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../test_redis_update_buffer.py               |  9 ++---
 .../proxy/db/test_db_spend_update_writer.py   | 35 ++++++++++---------
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
index 817c86a1bdf..53acbd2a32b 100644
--- a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
+++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
@@ -1,5 +1,6 @@
 import json
 from datetime import datetime, timezone
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -278,8 +279,8 @@ async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_fail
         SpendUpdateQueue,
     )
 
-    member_key = "organization_id::org-1::user_id::user-1"
-    pod_json = json.dumps({"org_member_list_transactions": {member_key: 0.25}})
+    member_key: Final = "organization_id::org-1::user_id::user-1"
+    pod_json: Final = json.dumps({"org_member_list_transactions": {member_key: 0.25}})
     mock_redis_cache.async_lpop_pipeline = AsyncMock(
         return_value=[[pod_json, pod_json], None, None, None, None, None, None]
     )
@@ -290,7 +291,7 @@ async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_fail
     assert db_spend["org_member_list_transactions"] == {member_key: 0.5}
 
     mock_redis_cache.async_rpush_pipeline = AsyncMock(side_effect=ConnectionError("redis went away"))
-    spend_queue = SpendUpdateQueue()
+    spend_queue: Final = SpendUpdateQueue()
     await spend_queue.add_update(
         {
             "entity_type": Litellm_EntityType.ORGANIZATION_MEMBER,
@@ -307,7 +308,7 @@ async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_fail
         daily_agent_spend_update_queue=DailySpendUpdateQueue(),
     )
 
-    restored_spend = await spend_queue.flush_and_get_aggregated_db_spend_update_transactions()
+    restored_spend: Final = await spend_queue.flush_and_get_aggregated_db_spend_update_transactions()
     assert restored_spend["org_member_list_transactions"] == {member_key: 1.5}
 
 
diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
index 64001146e05..8cbb415ec58 100644
--- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
+++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
@@ -8,6 +8,7 @@ from collections.abc import Callable
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from types import SimpleNamespace
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock, call, patch
 
 import pytest
@@ -949,19 +950,19 @@ async def test_org_spend_increments_organization_membership_row_for_the_calling_
     """A request made with a user_id inside an org must increment that user's
     LiteLLM_OrganizationMembership.spend, not only the org total, or the
     Organizations > Members UI renders '-' for every member."""
-    db_writer = DBSpendUpdateWriter()
+    db_writer: Final = DBSpendUpdateWriter()
     await db_writer._update_org_db(
         response_cost=0.75,
         org_id="org-abc",
         user_id="user-xyz",
         prisma_client=MagicMock(),
     )
-    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+    transactions: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
 
-    mock_batcher = MagicMock()
-    mock_prisma_client = MagicMock()
+    mock_batcher: Final = MagicMock()
+    mock_prisma_client: Final = MagicMock()
     mock_prisma_client.db.tx = MagicMock(return_value=_good_tx(mock_batcher))
-    proxy_logging = MagicMock()
+    proxy_logging: Final = MagicMock()
     proxy_logging.call_details = {}
 
     await db_writer._commit_spend_updates_to_db(
@@ -983,19 +984,19 @@ async def test_org_spend_increments_organization_membership_row_for_the_calling_
 
 @pytest.mark.asyncio
 async def test_org_spend_without_user_id_leaves_organization_membership_untouched():
-    db_writer = DBSpendUpdateWriter()
+    db_writer: Final = DBSpendUpdateWriter()
     await db_writer._update_org_db(
         response_cost=0.75,
         org_id="org-abc",
         user_id=None,
         prisma_client=MagicMock(),
     )
-    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+    transactions: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
 
-    mock_batcher = MagicMock()
-    mock_prisma_client = MagicMock()
+    mock_batcher: Final = MagicMock()
+    mock_prisma_client: Final = MagicMock()
     mock_prisma_client.db.tx = MagicMock(return_value=_good_tx(mock_batcher))
-    proxy_logging = MagicMock()
+    proxy_logging: Final = MagicMock()
     proxy_logging.call_details = {}
 
     await db_writer._commit_spend_updates_to_db(
@@ -1011,19 +1012,19 @@ async def test_org_spend_without_user_id_leaves_organization_membership_untouche
 
 @pytest.mark.asyncio
 async def test_org_spend_keeps_member_attribution_when_ids_contain_the_key_delimiter():
-    db_writer = DBSpendUpdateWriter()
+    db_writer: Final = DBSpendUpdateWriter()
     await db_writer._update_org_db(
         response_cost=0.75,
         org_id="division::west",
         user_id="user::42",
         prisma_client=MagicMock(),
     )
-    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+    transactions: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
 
-    mock_batcher = MagicMock()
-    mock_prisma_client = MagicMock()
+    mock_batcher: Final = MagicMock()
+    mock_prisma_client: Final = MagicMock()
     mock_prisma_client.db.tx = MagicMock(return_value=_good_tx(mock_batcher))
-    proxy_logging = MagicMock()
+    proxy_logging: Final = MagicMock()
     proxy_logging.call_details = {}
 
     await db_writer._commit_spend_updates_to_db(
@@ -1041,7 +1042,7 @@ async def test_org_spend_keeps_member_attribution_when_ids_contain_the_key_delim
 
 @pytest.mark.asyncio
 async def test_batch_database_updates_queues_org_member_spend_for_the_request_user():
-    db_writer = DBSpendUpdateWriter()
+    db_writer: Final = DBSpendUpdateWriter()
     await db_writer._batch_database_updates(
         response_cost=0.1,
         user_id="u1",
@@ -1053,7 +1054,7 @@ async def test_batch_database_updates_queues_org_member_spend_for_the_request_us
         litellm_proxy_budget_name=None,
         payload={"request_id": "req-1", "model": "gpt-4o-mini", "spend": 0.1},
     )
-    transactions = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
+    transactions: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
 
     assert transactions["org_list_transactions"] == {"org1": 0.1}
     assert transactions["org_member_list_transactions"] == {"organization_id::org1::user_id::u1": 0.1}

From 2505fcf95e6470bac5cdddb65d63b143c3605937 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 20:05:23 +0000
Subject: [PATCH 20/67] test(proxy): type the fixture parameters of the org
 member redis test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/db/db_transaction_queue/test_redis_update_buffer.py   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
index 53acbd2a32b..cc8b10150bd 100644
--- a/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
+++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_redis_update_buffer.py
@@ -269,7 +269,7 @@ async def test_get_all_transactions_from_redis_buffer_pipeline(redis_update_buff
 
 @pytest.mark.asyncio
 async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_failure(
-    redis_update_buffer, mock_redis_cache
+    redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock
 ):
     from litellm.proxy._types import Litellm_EntityType
     from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (

From e54b93017ba37e49948e7d01f4f4d794bb913a3f Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 15 Sep 2026 13:16:50 -0700
Subject: [PATCH 21/67] fix(jwt-auth): scope JWT key mappings by issuer to
 prevent cross-issuer collisions

---
 .../migration.sql                             |  18 ++
 .../litellm_proxy_extras/schema.prisma        |   8 +-
 litellm/proxy/_lazy_openapi_snapshot.json     |  33 ++++
 litellm/proxy/_types.py                       |   3 +
 litellm/proxy/auth/auth_checks.py             |  26 ++-
 litellm/proxy/auth/user_api_key_auth.py       |  55 +++++-
 .../jwt_key_mapping_endpoints.py              |  25 ++-
 litellm/proxy/schema.prisma                   |   8 +-
 schema.prisma                                 |   8 +-
 .../proxy_unit_tests/test_jwt_key_mapping.py  | 180 ++++++++++++++++++
 .../proxy/auth/test_user_api_key_auth.py      | 179 ++++++++++++++++-
 .../test_key_management_endpoints.py          |  15 +-
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   6 +
 13 files changed, 527 insertions(+), 37 deletions(-)
 create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_scope_jwt_key_mapping_by_issuer/migration.sql

diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_scope_jwt_key_mapping_by_issuer/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_scope_jwt_key_mapping_by_issuer/migration.sql
new file mode 100644
index 00000000000..c9572066ab6
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_scope_jwt_key_mapping_by_issuer/migration.sql
@@ -0,0 +1,18 @@
+-- DropIndex
+DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_is_act_idx";
+
+-- DropIndex
+DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_key";
+
+-- AlterTable
+-- NOT NULL DEFAULT '' (not nullable): Postgres unique constraints treat every
+-- NULL as distinct, so a nullable column would let multiple unscoped mappings
+-- collide on the same claim without a constraint violation. The constant
+-- default is a fast, metadata-only backfill for existing rows, not a rewrite.
+ALTER TABLE "LiteLLM_JWTKeyMapping" ADD COLUMN IF NOT EXISTS "jwt_issuer" TEXT NOT NULL DEFAULT '';
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_idx" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value", "is_active");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_key" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value");
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index 8072df5aa5b..62853d8e4b8 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -487,6 +487,10 @@ model LiteLLM_VerificationToken {
 
 model LiteLLM_JWTKeyMapping {
     id                String   @id @default(uuid())
+    jwt_issuer        String   @default("")  // Scopes the mapping to one configured issuer; "" matches any issuer.
+                                              // Not nullable: Postgres unique constraints treat every NULL as
+                                              // distinct, so a nullable column would let multiple unscoped
+                                              // mappings collide on the same claim without a constraint violation.
     jwt_claim_name    String   // e.g. "sub", "email"
     jwt_claim_value   String   // The claim value to match
     token             String   // Hashed virtual key (FK)
@@ -499,8 +503,8 @@ model LiteLLM_JWTKeyMapping {
 
     litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade)
 
-    @@unique([jwt_claim_name, jwt_claim_value])
-    @@index([jwt_claim_name, jwt_claim_value, is_active])
+    @@unique([jwt_issuer, jwt_claim_name, jwt_claim_value])
+    @@index([jwt_issuer, jwt_claim_name, jwt_claim_value, is_active])
 }
 
 // Deprecated keys during grace period - allows old key to work until revoke_at
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index f3b579d22c7..c5d1e7e8ece 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -15226,6 +15226,17 @@
               "title": "Jwt Claim Value",
               "type": "string"
             },
+            "jwt_issuer": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Jwt Issuer"
+            },
             "key": {
               "title": "Key",
               "type": "string"
@@ -15310,6 +15321,17 @@
               "title": "Jwt Claim Value",
               "type": "string"
             },
+            "jwt_issuer": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Jwt Issuer"
+            },
             "updated_at": {
               "format": "date-time",
               "title": "Updated At",
@@ -15366,6 +15388,17 @@
               ],
               "title": "Is Active"
             },
+            "jwt_issuer": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Jwt Issuer"
+            },
             "key": {
               "anyOf": [
                 {
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index afc2150934e..9e71a54c5ba 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -4485,12 +4485,14 @@ class CreateJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
     jwt_claim_name: str
     jwt_claim_value: str
     key: str
+    jwt_issuer: str | None = None
     description: str | None = None
 
 
 class UpdateJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
     id: str
     key: str | None = None
+    jwt_issuer: str | None = None
     description: str | None = None
     is_active: bool | None = None
 
@@ -4501,6 +4503,7 @@ class DeleteJWTKeyMappingRequest(LiteLLMPydanticObjectBase):
 
 class JWTKeyMappingResponse(LiteLLMPydanticObjectBase):
     id: str
+    jwt_issuer: str | None = None
     jwt_claim_name: str
     jwt_claim_value: str
     description: str | None = None
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 355fc3f6a21..e3783c94dc7 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -148,6 +148,7 @@ class _PrismaDictableRow(Protocol):
 
 class _PrismaJWTKeyMappingRow(Protocol):
     token: str
+    jwt_issuer: str
     jwt_claim_name: str
     jwt_claim_value: str
 
@@ -3601,9 +3602,18 @@ async def _fetch_key_object_from_db_with_reconnect(
             raise
 
 
-def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str) -> str:
-    """Cache key under which ``_resolve_jwt_to_virtual_key`` stores a JWT-claim-to-key mapping."""
-    return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}"
+def jwt_key_mapping_cache_key(jwt_claim_name: str, jwt_claim_value: str, jwt_issuer: str | None = None) -> str:
+    """Cache key under which a JWT-claim-to-key mapping is stored, scoped to one
+    issuer (or the issuer-agnostic/global scope when ``jwt_issuer`` is falsy).
+
+    Scoped by issuer (when one is configured) so a cached hit or ``__NO_MAPPING__`` miss
+    for one issuer's claim value can never be served to a different issuer whose claim
+    value happens to collide. Unchanged for the global scope, keeping the single-issuer
+    (no ``litellm_jwtauth.issuers`` configured) cache key format stable across this fix.
+    """
+    if not jwt_issuer:
+        return f"jwt_key_mapping:{jwt_claim_name}:{jwt_claim_value}"
+    return f"jwt_key_mapping:{jwt_issuer}:{jwt_claim_name}:{jwt_claim_value}"
 
 
 @log_db_metrics
@@ -3615,7 +3625,7 @@ async def get_jwt_key_mapping_cache_keys_for_token(
     mappings: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_many(
         where={"token": hashed_token}
     )
-    return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value) for m in mappings)
+    return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value, m.jwt_issuer) for m in mappings)
 
 
 @log_db_metrics
@@ -3623,9 +3633,14 @@ async def get_jwt_key_mapping_object(
     jwt_claim_name: str,
     jwt_claim_value: str,
     prisma_client: PrismaClient,
+    jwt_issuer: str | None = None,
 ) -> str | None:
     """
-    Lookup a JWT-to-virtual-key mapping from the database.
+    Lookup a JWT-to-virtual-key mapping from the database for one exact scope:
+    ``jwt_issuer`` (or the global/issuer-agnostic scope when falsy). Does not fall
+    back to the global scope itself -- a caller that wants "issuer-scoped mapping,
+    else the global one" queries both scopes itself, so each result can be cached
+    under its own scope's key (see ``_resolve_jwt_to_virtual_key``).
 
     Returns the hashed token (str) if a matching active mapping is found, else None.
     """
@@ -3633,6 +3648,7 @@ async def get_jwt_key_mapping_object(
         where={
             "jwt_claim_name": jwt_claim_name,
             "jwt_claim_value": jwt_claim_value,
+            "jwt_issuer": jwt_issuer or "",
             "is_active": True,
         }
     )
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 7d62baf39a8..1beacae819f 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -269,6 +269,17 @@ class _TokenTeamModels(Protocol):
     def team_models(self) -> list[str]: ...
 
 
+class _RawCacheRead(Protocol):
+    async def async_get_cache(self, *, key: str) -> object: ...
+
+
+def _raw_cache(cache: _RawCacheRead) -> _RawCacheRead:
+    """View an untyped cache object's ``async_get_cache`` as returning ``object``
+    instead of ``Any``, so a caller can ``isinstance``-narrow it without paying
+    the ``reportAny`` cost of the underlying (unannotated) cache implementation."""
+    return cache
+
+
 def _token_team_models(valid_token: _TokenTeamModels) -> list[str]:
     return valid_token.team_models
 
@@ -842,6 +853,7 @@ class _PendingAutoRegister(NamedTuple):
     claim_field: str
     claim_value: str
     cache_key: str
+    jwt_issuer: str | None = None
 
 
 async def _auto_register_jwt_mapping(
@@ -853,6 +865,7 @@ async def _auto_register_jwt_mapping(
     parent_otel_span: Span | None,
     proxy_logging_obj: ProxyLogging,
     cache_key: str,
+    jwt_issuer: str | None = None,
     team_id: str | None = None,
     user_id: str | None = None,
     org_id: str | None = None,
@@ -905,6 +918,7 @@ async def _auto_register_jwt_mapping(
     try:
         await prisma_client.db.litellm_jwtkeymapping.create(
             data={
+                "jwt_issuer": jwt_issuer or "",
                 "jwt_claim_name": virtual_key_claim_field,
                 "jwt_claim_value": claim_value,
                 "token": token_hash,
@@ -939,6 +953,7 @@ async def _auto_register_jwt_mapping(
                 jwt_claim_name=virtual_key_claim_field,
                 jwt_claim_value=claim_value,
                 prisma_client=prisma_client,
+                jwt_issuer=jwt_issuer,
             )
             if token_hash is None:
                 # The winner's mapping vanished between the unique-constraint
@@ -1041,7 +1056,7 @@ async def _resolve_jwt_to_virtual_key(
             )
         return None
 
-    cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value))
+    cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value), normalized_issuer)
     raw_cached_mapping: Final = await user_api_key_cache.async_get_cache(cache_key)
     sentinel_written_by_this_policy: Final = behavior == UnregisteredJWTClientBehavior.AUTO_REGISTER
     cached_mapping: Final = (
@@ -1081,6 +1096,7 @@ async def _resolve_jwt_to_virtual_key(
                 claim_field=virtual_key_claim_field,
                 claim_value=str(claim_value),
                 cache_key=cache_key,
+                jwt_issuer=normalized_issuer,
             )
         return None
     elif cached_mapping is not None:
@@ -1094,21 +1110,44 @@ async def _resolve_jwt_to_virtual_key(
         )
 
     # Resolve the mapping from DB, or treat prisma_client=None as a definitive
-    # miss (no DB → no mapping can exist → apply no-match policy below).
+    # miss (no DB → no mapping can exist → apply no-match policy below). An
+    # issuer-scoped row wins; falling back to the global (no-issuer) row keeps
+    # mappings created before issuer scoping existed working for every issuer.
+    # Each tier is cached under ITS OWN key (the global tier under the
+    # issuer-less cache key, not under `cache_key`/this issuer's key) so that
+    # updating or deleting either row invalidates exactly the cache entries it
+    # can affect. Caching a global-row hit under the requesting issuer's key
+    # would leave every OTHER issuer that had fallen back to that same global
+    # mapping serving its stale token until TTL after the row changes.
+    ttl: Final = jwt_handler.litellm_jwtauth.virtual_key_mapping_cache_ttl
     token_hash: str | None = None
     if prisma_client is not None:
         token_hash = await get_jwt_key_mapping_object(
             jwt_claim_name=virtual_key_claim_field,
             jwt_claim_value=str(claim_value),
             prisma_client=prisma_client,
+            jwt_issuer=normalized_issuer,
         )
+        if token_hash is not None:
+            await user_api_key_cache.async_set_cache(key=cache_key, value=token_hash, ttl=ttl)
+        elif normalized_issuer is not None:
+            # Another issuer may have already resolved (and cached) this same
+            # global mapping -- check its cache entry before re-querying the DB.
+            global_cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value))
+            cached_global: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=global_cache_key)
+            if isinstance(cached_global, str) and cached_global != "__NO_MAPPING__":
+                token_hash = cached_global
+            else:
+                token_hash = await get_jwt_key_mapping_object(
+                    jwt_claim_name=virtual_key_claim_field,
+                    jwt_claim_value=str(claim_value),
+                    prisma_client=prisma_client,
+                    jwt_issuer=None,
+                )
+                if token_hash is not None:
+                    await user_api_key_cache.async_set_cache(key=global_cache_key, value=token_hash, ttl=ttl)
 
     if token_hash is not None:
-        await user_api_key_cache.async_set_cache(
-            key=cache_key,
-            value=token_hash,
-            ttl=jwt_handler.litellm_jwtauth.virtual_key_mapping_cache_ttl,
-        )
         return IdentityStore.key_from_principal(
             await IdentityStore(
                 prisma_client,
@@ -1149,6 +1188,7 @@ async def _resolve_jwt_to_virtual_key(
             claim_field=virtual_key_claim_field,
             claim_value=str(claim_value),
             cache_key=cache_key,
+            jwt_issuer=normalized_issuer,
         )
 
     # FALLBACK_TEAM_MAPPING (default): cache the miss and return None so the
@@ -1641,6 +1681,7 @@ async def _user_api_key_auth_builder(
                             parent_otel_span=parent_otel_span,
                             proxy_logging_obj=proxy_logging_obj,
                             cache_key=pending_auto_register.cache_key,
+                            jwt_issuer=pending_auto_register.jwt_issuer,
                             team_id=team_id,
                             user_id=user_id,
                             org_id=org_id,
diff --git a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
index 694930a543c..07234883062 100644
--- a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
+++ b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
@@ -28,6 +28,9 @@ class _JWTKeyMappingRecord(Protocol):
     @property
     def id(self) -> str: ...
 
+    @property
+    def jwt_issuer(self) -> str: ...
+
     @property
     def jwt_claim_name(self) -> str: ...
 
@@ -78,6 +81,7 @@ def _to_response(mapping: _JWTKeyMappingRecord) -> JWTKeyMappingResponse:
     """Convert a Prisma mapping object to a safe response (no hashed token)."""
     return JWTKeyMappingResponse(
         id=mapping.id,
+        jwt_issuer=mapping.jwt_issuer or None,
         jwt_claim_name=mapping.jwt_claim_name,
         jwt_claim_value=mapping.jwt_claim_value,
         description=mapping.description,
@@ -109,6 +113,7 @@ async def create_jwt_key_mapping(
     try:
         hashed_key: Final = hash_token(data.key)
         create_data: Final = {
+            "jwt_issuer": data.jwt_issuer or "",
             "jwt_claim_name": data.jwt_claim_name,
             "jwt_claim_value": data.jwt_claim_value,
             "token": hashed_key,
@@ -120,7 +125,7 @@ async def create_jwt_key_mapping(
 
         new_mapping: Final = await _mapping_table(prisma_client).create(data=create_data)
 
-        cache_key: Final = jwt_key_mapping_cache_key(data.jwt_claim_name, data.jwt_claim_value)
+        cache_key: Final = jwt_key_mapping_cache_key(data.jwt_claim_name, data.jwt_claim_value, data.jwt_issuer)
         await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache)
 
         return _to_response(new_mapping)
@@ -131,7 +136,10 @@ async def create_jwt_key_mapping(
         if "unique" in error_str or "p2002" in error_str:
             raise HTTPException(
                 status_code=409,
-                detail=f"A mapping for claim '{data.jwt_claim_name}' = '{data.jwt_claim_value}' already exists.",
+                detail=(
+                    f"A mapping for claim '{data.jwt_claim_name}' = '{data.jwt_claim_value}' "
+                    f"already exists for issuer '{data.jwt_issuer}'."
+                ),
             )
         if "foreign" in error_str or "p2003" in error_str:
             raise HTTPException(
@@ -161,6 +169,9 @@ async def update_jwt_key_mapping(
     update_data: Final = data.model_dump(exclude_unset=True, exclude={"id", "key"})
     if data.key is not None:
         update_data["token"] = hash_token(data.key)
+    if "jwt_issuer" in update_data:
+        # DB column is NOT NULL (see schema.prisma); "" is the global/unscoped sentinel.
+        update_data["jwt_issuer"] = update_data["jwt_issuer"] or ""
     update_data["updated_by"] = user_api_key_dict.user_id
 
     try:
@@ -178,9 +189,11 @@ async def update_jwt_key_mapping(
         # Evict only after the write commits: a concurrent request between an
         # early eviction and the commit would re-cache the old mapping and keep
         # it authorized until TTL.
-        old_cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value)
+        old_cache_key: Final = jwt_key_mapping_cache_key(
+            old_mapping.jwt_claim_name, old_mapping.jwt_claim_value, old_mapping.jwt_issuer
+        )
         new_cache_key: Final = jwt_key_mapping_cache_key(
-            updated_mapping.jwt_claim_name, updated_mapping.jwt_claim_value
+            updated_mapping.jwt_claim_name, updated_mapping.jwt_claim_value, updated_mapping.jwt_issuer
         )
         cache_keys: Final = (old_cache_key,) if old_cache_key == new_cache_key else (old_cache_key, new_cache_key)
         await evict_and_broadcast(cache_keys=cache_keys, user_api_key_cache=user_api_key_cache)
@@ -227,7 +240,9 @@ async def delete_jwt_key_mapping(
 
         # Evict only after the row is gone, else a concurrent request can
         # re-cache the deleted mapping and keep it authorized until TTL.
-        cache_key: Final = jwt_key_mapping_cache_key(old_mapping.jwt_claim_name, old_mapping.jwt_claim_value)
+        cache_key: Final = jwt_key_mapping_cache_key(
+            old_mapping.jwt_claim_name, old_mapping.jwt_claim_value, old_mapping.jwt_issuer
+        )
         await evict_and_broadcast(cache_keys=(cache_key,), user_api_key_cache=user_api_key_cache)
         return {"status": "success"}
     except HTTPException:
diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 8072df5aa5b..62853d8e4b8 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -487,6 +487,10 @@ model LiteLLM_VerificationToken {
 
 model LiteLLM_JWTKeyMapping {
     id                String   @id @default(uuid())
+    jwt_issuer        String   @default("")  // Scopes the mapping to one configured issuer; "" matches any issuer.
+                                              // Not nullable: Postgres unique constraints treat every NULL as
+                                              // distinct, so a nullable column would let multiple unscoped
+                                              // mappings collide on the same claim without a constraint violation.
     jwt_claim_name    String   // e.g. "sub", "email"
     jwt_claim_value   String   // The claim value to match
     token             String   // Hashed virtual key (FK)
@@ -499,8 +503,8 @@ model LiteLLM_JWTKeyMapping {
 
     litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade)
 
-    @@unique([jwt_claim_name, jwt_claim_value])
-    @@index([jwt_claim_name, jwt_claim_value, is_active])
+    @@unique([jwt_issuer, jwt_claim_name, jwt_claim_value])
+    @@index([jwt_issuer, jwt_claim_name, jwt_claim_value, is_active])
 }
 
 // Deprecated keys during grace period - allows old key to work until revoke_at
diff --git a/schema.prisma b/schema.prisma
index 8072df5aa5b..62853d8e4b8 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -487,6 +487,10 @@ model LiteLLM_VerificationToken {
 
 model LiteLLM_JWTKeyMapping {
     id                String   @id @default(uuid())
+    jwt_issuer        String   @default("")  // Scopes the mapping to one configured issuer; "" matches any issuer.
+                                              // Not nullable: Postgres unique constraints treat every NULL as
+                                              // distinct, so a nullable column would let multiple unscoped
+                                              // mappings collide on the same claim without a constraint violation.
     jwt_claim_name    String   // e.g. "sub", "email"
     jwt_claim_value   String   // The claim value to match
     token             String   // Hashed virtual key (FK)
@@ -499,8 +503,8 @@ model LiteLLM_JWTKeyMapping {
 
     litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade)
 
-    @@unique([jwt_claim_name, jwt_claim_value])
-    @@index([jwt_claim_name, jwt_claim_value, is_active])
+    @@unique([jwt_issuer, jwt_claim_name, jwt_claim_value])
+    @@index([jwt_issuer, jwt_claim_name, jwt_claim_value, is_active])
 }
 
 // Deprecated keys during grace period - allows old key to work until revoke_at
diff --git a/tests/proxy_unit_tests/test_jwt_key_mapping.py b/tests/proxy_unit_tests/test_jwt_key_mapping.py
index e8db5d1cf7f..3f2c04336a7 100644
--- a/tests/proxy_unit_tests/test_jwt_key_mapping.py
+++ b/tests/proxy_unit_tests/test_jwt_key_mapping.py
@@ -91,6 +91,154 @@ async def test_jwt_to_virtual_key_mapping_resolution():
         prisma_client.db.litellm_jwtkeymapping.find_first.assert_not_called()
 
 
+@pytest.mark.asyncio
+async def test_colliding_claim_value_from_another_issuer_does_not_resolve_to_the_wrong_virtual_key():
+    """LIT-7417: a mapping registered for one issuer must not answer a lookup from a
+    DIFFERENT issuer whose claim value happens to collide, even though both issuers
+    map the same claim field (``sub``) to a virtual key."""
+    issuer_a = "https://issuer-a.example.com"
+    issuer_b = "https://issuer-b.example.com"
+
+    jwt_handler = JWTHandler()
+    jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(
+        virtual_key_claim_field="sub", virtual_key_mapping_cache_ttl=3600
+    )
+
+    rows = [
+        {
+            "jwt_issuer": issuer_b,
+            "jwt_claim_name": "sub",
+            "jwt_claim_value": "dev-alice",
+            "token": "hashed-issuer-b-key",
+            "is_active": True,
+        }
+    ]
+
+    async def fake_find_first(where):
+        for row in rows:
+            if all(row.get(k) == v for k, v in where.items()):
+                return MagicMock(**row)
+        return None
+
+    prisma_client = MagicMock()
+    prisma_client.db.litellm_jwtkeymapping.find_first = AsyncMock(side_effect=fake_find_first)
+
+    # Dependency-inject the resolved key via the cache (IdentityStore._resolve_key
+    # reads it from here) instead of monkeypatching IdentityStore itself.
+    user_api_key_cache = DualCache()
+    await user_api_key_cache.async_set_cache(
+        key="hashed-issuer-b-key",
+        value=UserAPIKeyAuth(token="hashed-issuer-b-key", team_id="issuer-b-team"),
+    )
+
+    # The rightful owner: issuer-b's own claim resolves to its mapping.
+    owner_result = await _resolve_jwt_to_virtual_key(
+        jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: issuer_b, "sub": "dev-alice"},
+        jwt_handler=jwt_handler,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=None,
+        proxy_logging_obj=None,
+    )
+    assert isinstance(owner_result, UserAPIKeyAuth)
+    assert owner_result.token == "hashed-issuer-b-key"
+
+    # A validly-signed token from issuer-a carrying the SAME claim value must not
+    # inherit issuer-b's mapping. Default behavior is fallback_team_mapping, so a
+    # correctly-scoped miss returns None instead of resolving to issuer-b's key.
+    colliding_result = await _resolve_jwt_to_virtual_key(
+        jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: issuer_a, "sub": "dev-alice"},
+        jwt_handler=jwt_handler,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=None,
+        proxy_logging_obj=None,
+    )
+    assert colliding_result is None
+
+
+@pytest.mark.asyncio
+async def test_global_mapping_resolution_is_cached_under_the_global_key_not_the_requesting_issuer():
+    """LIT-7417: caching a global (unscoped) mapping's hit under the REQUESTING
+    issuer's key would leave every issuer that falls back to it holding its own
+    stale copy after the row is updated/deleted -- CRUD only evicts the cache key
+    computed from the row's own scope (global), so a copy cached under some other
+    issuer's key would keep resolving to the old token until TTL. Caching it under
+    the global key instead means every issuer shares (and CRUD correctly evicts)
+    the exact same entry."""
+    issuer_a = "https://issuer-a.example.com"
+    issuer_b = "https://issuer-b.example.com"
+
+    jwt_handler = JWTHandler()
+    jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth(
+        issuers=[
+            {
+                "issuer": issuer_a,
+                "jwks_url": f"{issuer_a}/jwks",
+                "virtual_key_claim_field": "sub",
+                "disable_audience_validation": True,
+            },
+            {
+                "issuer": issuer_b,
+                "jwks_url": f"{issuer_b}/jwks",
+                "virtual_key_claim_field": "sub",
+                "disable_audience_validation": True,
+            },
+        ]
+    )
+
+    rows = [
+        {
+            "jwt_issuer": "",
+            "jwt_claim_name": "sub",
+            "jwt_claim_value": "legacy-user",
+            "token": "hashed-legacy-key",
+            "is_active": True,
+        }
+    ]
+
+    async def fake_find_first(where):
+        for row in rows:
+            if all(row.get(k) == v for k, v in where.items()):
+                return MagicMock(**row)
+        return None
+
+    prisma_client = MagicMock()
+    find_first = AsyncMock(side_effect=fake_find_first)
+    prisma_client.db.litellm_jwtkeymapping.find_first = find_first
+
+    user_api_key_cache = DualCache()
+    await user_api_key_cache.async_set_cache(
+        key="hashed-legacy-key",
+        value=UserAPIKeyAuth(token="hashed-legacy-key", team_id="legacy-team"),
+    )
+
+    resolved_a = await _resolve_jwt_to_virtual_key(
+        jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: issuer_a, "sub": "legacy-user"},
+        jwt_handler=jwt_handler,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=None,
+        proxy_logging_obj=None,
+    )
+    assert isinstance(resolved_a, UserAPIKeyAuth)
+    assert find_first.await_count == 2  # issuer-a-scoped miss, then global hit
+
+    # issuer-b resolving the SAME global mapping must hit the cache issuer-a's
+    # resolution populated, not issue a fresh DB query for the global row again.
+    resolved_b = await _resolve_jwt_to_virtual_key(
+        jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: issuer_b, "sub": "legacy-user"},
+        jwt_handler=jwt_handler,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=None,
+        proxy_logging_obj=None,
+    )
+    assert isinstance(resolved_b, UserAPIKeyAuth)
+    assert resolved_b.token == "hashed-legacy-key"
+    assert find_first.await_count == 3  # +1 for issuer-b's own issuer-scoped miss; global tier served from cache
+
+
 @pytest.mark.asyncio
 async def test_jwt_to_virtual_key_mapping_no_mapping():
     """
@@ -223,6 +371,7 @@ def test_to_response_excludes_token():
     now = datetime.now(timezone.utc)
     mock_mapping = MagicMock()
     mock_mapping.id = "mapping-1"
+    mock_mapping.jwt_issuer = None
     mock_mapping.jwt_claim_name = "email"
     mock_mapping.jwt_claim_value = "user@example.com"
     mock_mapping.token = "hashed_secret_value"
@@ -275,10 +424,12 @@ def _mock_mapping(
     id="mapping-1",
     claim_name="email",
     claim_value="user@example.com",
+    issuer=None,
 ):
     now = datetime.now(timezone.utc)
     m = MagicMock()
     m.id = id
+    m.jwt_issuer = issuer
     m.jwt_claim_name = claim_name
     m.jwt_claim_value = claim_value
     m.token = "hashed_token"
@@ -485,6 +636,35 @@ async def test_create_success_returns_response_without_token():
         assert result.jwt_claim_name == "email"
 
 
+@pytest.mark.asyncio
+async def test_create_without_issuer_stores_empty_string_not_null():
+    """LIT-7417: the DB column is NOT NULL (see schema.prisma). Storing a real NULL
+    for an unscoped mapping would let Postgres accept unlimited duplicate unscoped
+    rows for the same claim (NULL is never equal to NULL in a unique constraint),
+    so two mappings for the same claim value could point at two different keys with
+    no conflict, and resolution would pick whichever one Postgres returns first."""
+    from litellm.proxy._types import CreateJWTKeyMappingRequest
+
+    mock_prisma = _mock_prisma()
+    mock_prisma.db.litellm_jwtkeymapping.create.return_value = _mock_mapping()
+    mock_cache = AsyncMock()
+
+    data = CreateJWTKeyMappingRequest(jwt_claim_name="sub", jwt_claim_value="dev-alice", key="sk-test-key")
+
+    with (
+        patch(  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+            "litellm.proxy.proxy_server.prisma_client", mock_prisma
+        ),
+        patch(  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+            "litellm.proxy.proxy_server.user_api_key_cache", mock_cache
+        ),
+    ):
+        await create_jwt_key_mapping(data=data, user_api_key_dict=_make_admin_auth())
+
+    sent_data = mock_prisma.db.litellm_jwtkeymapping.create.call_args.kwargs["data"]
+    assert sent_data["jwt_issuer"] == ""
+
+
 # ──────────────────────────────────────────────
 # Tests: unregistered_jwt_client_behavior
 # ──────────────────────────────────────────────
diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
index 866ea0b20e4..bd7ff62ac8b 100644
--- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
+++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
@@ -32,7 +32,13 @@ from litellm.proxy._types import (
     JWTRoutingOverride,
 )
 from litellm.proxy.auth.handle_jwt import JWTHandler
-from litellm.proxy.auth.auth_checks import TeamNotFoundError, UserNotFoundError, get_key_object, _cache_key_object
+from litellm.proxy.auth.auth_checks import (
+    TeamNotFoundError,
+    UserNotFoundError,
+    get_key_object,
+    _cache_key_object,
+    jwt_key_mapping_cache_key,
+)
 from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.auth.user_api_key_auth import (
     _check_key_model_budget_with_fallback,
@@ -7948,13 +7954,38 @@ def _per_issuer_virtual_key_jwt_handler(
 
 
 def _fake_prisma_with_jwt_key_mapping(hashed_token: str | None) -> tuple[SimpleNamespace, AsyncMock]:
+    """Every ``find_first`` call (issuer-scoped or global fallback) resolves the same way."""
     find_first = AsyncMock(return_value=None if hashed_token is None else SimpleNamespace(token=hashed_token))
     prisma_client = SimpleNamespace(db=SimpleNamespace(litellm_jwtkeymapping=SimpleNamespace(find_first=find_first)))
     return prisma_client, find_first
 
 
-def _mapping_where(claim_name: str, claim_value: str) -> dict[str, str | bool]:
-    return {"jwt_claim_name": claim_name, "jwt_claim_value": claim_value, "is_active": True}
+def _fake_prisma_jwt_key_mapping_table(rows: list[dict[str, object]]) -> tuple[SimpleNamespace, AsyncMock]:
+    """A ``find_first`` whose result depends on the ``where`` clause, like a real table.
+
+    Matches a row when every key present in ``where`` equals that key on the row --
+    a key ``get_jwt_key_mapping_object`` omits (e.g. old, issuer-blind code never
+    sending ``jwt_issuer``) does not constrain the match, exactly like Prisma.
+    """
+
+    async def _find_first(where: dict[str, object]) -> SimpleNamespace | None:
+        for row in rows:
+            if all(row.get(k) == v for k, v in where.items()):
+                return SimpleNamespace(**row)
+        return None
+
+    find_first = AsyncMock(side_effect=_find_first)
+    prisma_client = SimpleNamespace(db=SimpleNamespace(litellm_jwtkeymapping=SimpleNamespace(find_first=find_first)))
+    return prisma_client, find_first
+
+
+def _mapping_where(claim_name: str, claim_value: str, jwt_issuer: str | None) -> dict[str, str | bool]:
+    return {
+        "jwt_claim_name": claim_name,
+        "jwt_claim_value": claim_value,
+        "jwt_issuer": jwt_issuer or "",
+        "is_active": True,
+    }
 
 
 @pytest.mark.asyncio
@@ -7978,11 +8009,13 @@ async def test_per_issuer_virtual_key_claim_field_selects_the_issuer_mapping_for
         proxy_logging_obj=MagicMock(),
     )
 
-    find_first.assert_awaited_once_with(where=_mapping_where("sub", "svc-account-7"))
+    # Issuer-scoped lookup hits on the first query, so no global fallback query runs.
+    find_first.assert_awaited_once_with(where=_mapping_where("sub", "svc-account-7", ISSUER_TWO))
     assert isinstance(resolved, UserAPIKeyAuth)
     assert resolved.token == "hashed-mapped-key"
     assert resolved.team_id == "svc-team"
-    assert await user_api_key_cache.async_get_cache("jwt_key_mapping:sub:svc-account-7") == "hashed-mapped-key"
+    cache_key = jwt_key_mapping_cache_key("sub", "svc-account-7", ISSUER_TWO)
+    assert await user_api_key_cache.async_get_cache(cache_key) == "hashed-mapped-key"
 
 
 @pytest.mark.asyncio
@@ -8015,7 +8048,11 @@ async def test_per_issuer_reject_behavior_does_not_leak_into_the_team_issuer():
 
     assert exc.value.status_code == 403
     assert "No registered mapping for sub='unknown-svc'" in str(exc.value.detail)
-    find_first.assert_awaited_once_with(where=_mapping_where("sub", "unknown-svc"))
+    # REJECT checks the issuer-scoped row first, then falls back to a global (NULL-issuer) row.
+    assert [c.kwargs["where"] for c in find_first.await_args_list] == [
+        _mapping_where("sub", "unknown-svc", ISSUER_TWO),
+        _mapping_where("sub", "unknown-svc", None),
+    ]
 
 
 @pytest.mark.asyncio
@@ -8025,7 +8062,10 @@ async def test_proxy_admin_sentinel_cached_by_another_issuer_does_not_bypass_rej
     jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="sub", global_behavior="auto_register")
     prisma_client, find_first = _fake_prisma_with_jwt_key_mapping(None)
     user_api_key_cache = DualCache()
-    await user_api_key_cache.async_set_cache(key="jwt_key_mapping:sub:admin-7", value=_JWT_PROXY_ADMIN_SENTINEL)
+    # Sentinel cached under issuer-one's own key -- must never answer issuer-two's lookup.
+    await user_api_key_cache.async_set_cache(
+        key=jwt_key_mapping_cache_key("sub", "admin-7", ISSUER_ONE), value=_JWT_PROXY_ADMIN_SENTINEL
+    )
 
     auto_register_issuer_result = await _resolve_jwt_to_virtual_key(
         jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "admin-7"},
@@ -8050,7 +8090,10 @@ async def test_proxy_admin_sentinel_cached_by_another_issuer_does_not_bypass_rej
 
     assert exc.value.status_code == 403
     assert "No registered mapping for sub='admin-7'" in str(exc.value.detail)
-    find_first.assert_awaited_once_with(where=_mapping_where("sub", "admin-7"))
+    assert [c.kwargs["where"] for c in find_first.await_args_list] == [
+        _mapping_where("sub", "admin-7", ISSUER_TWO),
+        _mapping_where("sub", "admin-7", None),
+    ]
 
 
 @pytest.mark.asyncio
@@ -8079,7 +8122,125 @@ async def test_issuer_without_virtual_key_claim_field_falls_back_to_the_global_f
 
     assert with_claim is None
     assert without_claim is None
-    find_first.assert_awaited_once_with(where=_mapping_where("client_id", "app-9"))
+    # without_claim has no claim value and returns before ever reaching the DB.
+    assert [c.kwargs["where"] for c in find_first.await_args_list] == [
+        _mapping_where("client_id", "app-9", ISSUER_ONE),
+        _mapping_where("client_id", "app-9", None),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_colliding_claim_value_from_another_issuer_does_not_resolve_to_the_wrong_virtual_key():
+    """LIT-7417: a mapping registered for one issuer must not answer a lookup from a
+    DIFFERENT issuer whose claim value happens to collide, even though both issuers
+    use the same claim field (``sub``) for their virtual-key mapping."""
+    from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key
+
+    jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="sub")
+    prisma_client, find_first = _fake_prisma_jwt_key_mapping_table(
+        [
+            {
+                "jwt_issuer": ISSUER_TWO,
+                "jwt_claim_name": "sub",
+                "jwt_claim_value": "dev-alice",
+                "token": "hashed-issuer-b-key",
+                "is_active": True,
+            }
+        ]
+    )
+    user_api_key_cache = DualCache()
+    await user_api_key_cache.async_set_cache(
+        key="hashed-issuer-b-key",
+        value=UserAPIKeyAuth(token="hashed-issuer-b-key", api_key="hashed-issuer-b-key", team_id="issuer-b-team"),
+    )
+
+    owner_result = await _resolve_jwt_to_virtual_key(
+        jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_TWO, "sub": "dev-alice"},
+        jwt_handler=jwt_handler,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=None,
+        proxy_logging_obj=MagicMock(),
+    )
+    assert isinstance(owner_result, UserAPIKeyAuth)
+    assert owner_result.token == "hashed-issuer-b-key"
+
+    # issuer-one's behavior is fallback_team_mapping: a correctly-scoped miss must
+    # return None (fall through to team-based JWT auth), never issuer-two's key.
+    colliding_result = await _resolve_jwt_to_virtual_key(
+        jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "dev-alice"},
+        jwt_handler=jwt_handler,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=None,
+        proxy_logging_obj=MagicMock(),
+    )
+    assert colliding_result is None
+    assert find_first.await_count == 3  # owner hit (1 call) + colliding miss (issuer-scoped + global fallback)
+
+
+@pytest.mark.asyncio
+async def test_cached_resolution_for_one_issuer_does_not_leak_to_a_colliding_issuer():
+    """A cached positive resolution must be keyed by issuer too, or a colliding
+    claim value from another issuer could be served straight from cache without
+    ever reaching the (correctly issuer-scoped) DB lookup."""
+    from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key
+
+    jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="sub")
+    prisma_client, find_first = _fake_prisma_jwt_key_mapping_table([])
+    user_api_key_cache = DualCache()
+    await user_api_key_cache.async_set_cache(
+        key=jwt_key_mapping_cache_key("sub", "dev-alice", ISSUER_TWO), value="hashed-issuer-b-key"
+    )
+
+    colliding_result = await _resolve_jwt_to_virtual_key(
+        jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: ISSUER_ONE, "sub": "dev-alice"},
+        jwt_handler=jwt_handler,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=None,
+        proxy_logging_obj=MagicMock(),
+    )
+    assert colliding_result is None
+    # Must have gone to the DB rather than serving issuer-two's cached token.
+    assert find_first.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_issuer_agnostic_mapping_matches_every_issuer():
+    """A mapping created before issuer scoping existed (``jwt_issuer`` is NULL) keeps
+    matching any issuer, so existing global mappings are not broken by this fix."""
+    from litellm.proxy.auth.user_api_key_auth import _resolve_jwt_to_virtual_key
+
+    jwt_handler = _per_issuer_virtual_key_jwt_handler(global_claim_field="sub")
+    prisma_client, _find_first = _fake_prisma_jwt_key_mapping_table(
+        [
+            {
+                "jwt_issuer": "",
+                "jwt_claim_name": "sub",
+                "jwt_claim_value": "legacy-user",
+                "token": "hashed-legacy-key",
+                "is_active": True,
+            }
+        ]
+    )
+    user_api_key_cache = DualCache()
+    await user_api_key_cache.async_set_cache(
+        key="hashed-legacy-key",
+        value=UserAPIKeyAuth(token="hashed-legacy-key", api_key="hashed-legacy-key", team_id="legacy-team"),
+    )
+
+    for issuer in (ISSUER_ONE, ISSUER_TWO):
+        resolved = await _resolve_jwt_to_virtual_key(
+            jwt_claims={JWTHandler.LITELLM_JWT_ISSUER_CLAIM: issuer, "sub": "legacy-user"},
+            jwt_handler=jwt_handler,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            parent_otel_span=None,
+            proxy_logging_obj=MagicMock(),
+        )
+        assert isinstance(resolved, UserAPIKeyAuth)
+        assert resolved.token == "hashed-legacy-key"
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
index 63055872aa1..4e70063015d 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
@@ -36,7 +36,11 @@ from litellm.proxy._types import (
     UpdateKeyRequest,
 )
 from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
-from litellm.proxy.auth.auth_checks import _delete_cache_key_object, _project_cache_key
+from litellm.proxy.auth.auth_checks import (
+    _delete_cache_key_object,
+    _project_cache_key,
+    jwt_key_mapping_cache_key,
+)
 from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
 from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
@@ -5132,10 +5136,11 @@ async def test_delete_verification_tokens_persists_deleted_keys(monkeypatch):
 
 
 class _JWTMappingRow:
-    def __init__(self, token, jwt_claim_name, jwt_claim_value):
+    def __init__(self, token, jwt_claim_name, jwt_claim_value, jwt_issuer=None):
         self.token = token
         self.jwt_claim_name = jwt_claim_name
         self.jwt_claim_value = jwt_claim_value
+        self.jwt_issuer = jwt_issuer
 
 
 class _CascadingJWTMappingTable:
@@ -5226,7 +5231,7 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat
         ),
     )
 
-    assert recording_evict.cache_keys == ("jwt_key_mapping:email:user@example.com",)
+    assert recording_evict.cache_keys == (jwt_key_mapping_cache_key("email", "user@example.com", None),)
 
 
 @pytest.mark.asyncio
@@ -13131,11 +13136,11 @@ async def test_regenerate_evicts_jwt_key_mapping_cache_so_next_jwt_call_gets_new
         _execute_virtual_key_regeneration,
     )
 
-    stale_cache_key = "jwt_key_mapping:sub:user1"
+    stale_cache_key = jwt_key_mapping_cache_key("sub", "user1", None)
     existing_key = _make_regenerate_existing_key()
     mock_prisma_client = _make_regenerate_mock_prisma()
     mock_prisma_client.db.litellm_jwtkeymapping.find_many = AsyncMock(
-        return_value=[MagicMock(jwt_claim_name="sub", jwt_claim_value="user1")]
+        return_value=[MagicMock(jwt_claim_name="sub", jwt_claim_value="user1", jwt_issuer=None)]
     )
     mock_prisma_client.db.litellm_jwtkeymapping.find_first = AsyncMock(
         return_value=MagicMock(token="new-hashed-token")
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 4c16a8613eb..af3ee891cb4 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -27297,6 +27297,8 @@ export interface components {
             jwt_claim_name: string;
             /** Jwt Claim Value */
             jwt_claim_value: string;
+            /** Jwt Issuer */
+            jwt_issuer?: string | null;
             /** Key */
             key: string;
         };
@@ -28910,6 +28912,8 @@ export interface components {
             jwt_claim_name: string;
             /** Jwt Claim Value */
             jwt_claim_value: string;
+            /** Jwt Issuer */
+            jwt_issuer?: string | null;
             /**
              * Updated At
              * Format: date-time
@@ -38574,6 +38578,8 @@ export interface components {
             id: string;
             /** Is Active */
             is_active?: boolean | null;
+            /** Jwt Issuer */
+            jwt_issuer?: string | null;
             /** Key */
             key?: string | null;
         };

From 67c17b68fa6caf2d280021a744359b1d1f9a4300 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:09:01 +0000
Subject: [PATCH 22/67] refactor(jwt-auth): extract issuer-scoped mapping
 lookup to satisfy C901 budget

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/user_api_key_auth.py | 73 ++++++++++++++++---------
 1 file changed, 48 insertions(+), 25 deletions(-)

diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 1beacae819f..5958a68f975 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -998,6 +998,43 @@ async def _auto_register_jwt_mapping(
     return auto_registered_key
 
 
+async def _lookup_jwt_mapping_token_hash(
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+    virtual_key_claim_field: str,
+    claim_value: str,
+    normalized_issuer: str | None,
+    cache_key: str,
+    ttl: float,
+) -> str | None:
+    issuer_scoped: Final = await get_jwt_key_mapping_object(
+        jwt_claim_name=virtual_key_claim_field,
+        jwt_claim_value=claim_value,
+        prisma_client=prisma_client,
+        jwt_issuer=normalized_issuer,
+    )
+    if issuer_scoped is not None:
+        await user_api_key_cache.async_set_cache(key=cache_key, value=issuer_scoped, ttl=ttl)
+        return issuer_scoped
+    if normalized_issuer is None:
+        return None
+    # Another issuer may have already resolved (and cached) this same
+    # global mapping -- check its cache entry before re-querying the DB.
+    global_cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, claim_value)
+    cached_global: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=global_cache_key)
+    if isinstance(cached_global, str) and cached_global != "__NO_MAPPING__":
+        return cached_global
+    global_row: Final = await get_jwt_key_mapping_object(
+        jwt_claim_name=virtual_key_claim_field,
+        jwt_claim_value=claim_value,
+        prisma_client=prisma_client,
+        jwt_issuer=None,
+    )
+    if global_row is not None:
+        await user_api_key_cache.async_set_cache(key=global_cache_key, value=global_row, ttl=ttl)
+    return global_row
+
+
 async def _resolve_jwt_to_virtual_key(
     jwt_claims: dict,
     jwt_handler: JWTHandler,
@@ -1119,33 +1156,19 @@ async def _resolve_jwt_to_virtual_key(
     # can affect. Caching a global-row hit under the requesting issuer's key
     # would leave every OTHER issuer that had fallen back to that same global
     # mapping serving its stale token until TTL after the row changes.
-    ttl: Final = jwt_handler.litellm_jwtauth.virtual_key_mapping_cache_ttl
-    token_hash: str | None = None
-    if prisma_client is not None:
-        token_hash = await get_jwt_key_mapping_object(
-            jwt_claim_name=virtual_key_claim_field,
-            jwt_claim_value=str(claim_value),
+    token_hash: Final = (
+        await _lookup_jwt_mapping_token_hash(
             prisma_client=prisma_client,
-            jwt_issuer=normalized_issuer,
+            user_api_key_cache=user_api_key_cache,
+            virtual_key_claim_field=virtual_key_claim_field,
+            claim_value=str(claim_value),
+            normalized_issuer=normalized_issuer,
+            cache_key=cache_key,
+            ttl=jwt_handler.litellm_jwtauth.virtual_key_mapping_cache_ttl,
         )
-        if token_hash is not None:
-            await user_api_key_cache.async_set_cache(key=cache_key, value=token_hash, ttl=ttl)
-        elif normalized_issuer is not None:
-            # Another issuer may have already resolved (and cached) this same
-            # global mapping -- check its cache entry before re-querying the DB.
-            global_cache_key: Final = jwt_key_mapping_cache_key(virtual_key_claim_field, str(claim_value))
-            cached_global: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=global_cache_key)
-            if isinstance(cached_global, str) and cached_global != "__NO_MAPPING__":
-                token_hash = cached_global
-            else:
-                token_hash = await get_jwt_key_mapping_object(
-                    jwt_claim_name=virtual_key_claim_field,
-                    jwt_claim_value=str(claim_value),
-                    prisma_client=prisma_client,
-                    jwt_issuer=None,
-                )
-                if token_hash is not None:
-                    await user_api_key_cache.async_set_cache(key=global_cache_key, value=token_hash, ttl=ttl)
+        if prisma_client is not None
+        else None
+    )
 
     if token_hash is not None:
         return IdentityStore.key_from_principal(

From 9b3595434713e7ec26adc697e1f6fc1da3ae9ed2 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 15 Sep 2026 14:10:23 -0700
Subject: [PATCH 23/67] fix(ui): block usage export and flag the range when a
 spend page fails

The Usage page drains the daily activity endpoint page by page. A page that
threw was only logged to the console: the loading banner disappeared, the
partial totals stayed on screen looking final, and Export Data stayed
clickable, so the CSV handed to finance was silently short.

The hook now reports `failed`, PaginationStatusAlerts renders it as an error
banner naming how many pages actually loaded, and the export is blocked with
the reason on hover while the data on screen does not cover the range.
---
 .../_components/CacheLeakageCard.test.tsx     |  1 +
 .../_components/CostOptimizationView.tsx      |  1 +
 .../_components/PromptCachingTab.test.tsx     |  1 +
 .../_components/UsageTab.test.tsx             |  1 +
 .../_components/useDailyActivityRange.ts      |  4 +-
 .../components/EntityUsage/EntityUsage.tsx    |  9 +++
 .../_components/components/UsagePageView.tsx  | 24 +++++--
 .../hooks/usePaginatedDailyActivity.test.ts   | 66 +++++++++++++++++++
 .../hooks/usePaginatedDailyActivity.ts        |  8 ++-
 .../UsageExportHeader.test.tsx                | 21 ++++++
 .../EntityUsageExport/UsageExportHeader.tsx   | 13 ++--
 .../exportBlockedReason.test.ts               | 39 +++++++++++
 .../EntityUsageExport/exportBlockedReason.ts  | 21 ++++++
 .../shared/PaginationStatusAlerts.test.tsx    | 31 +++++++++
 .../shared/PaginationStatusAlerts.tsx         | 12 +++-
 .../KeySavingsTab.integration.test.tsx        |  1 +
 16 files changed, 242 insertions(+), 11 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts
 create mode 100644 ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx
index f320d8e0f97..54af13d8a90 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx
@@ -71,6 +71,7 @@ const renderWith = (results: DailyData[], overrides: Partial<DailyActivityRange>
         isFetchingMore: false,
         progress: { currentPage: 1, totalPages: 1 },
         cancelled: false,
+        failed: false,
         cancel: vi.fn(),
         ...overrides,
       }}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx
index 8094fa2e8b6..25bd3de0382 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx
@@ -87,6 +87,7 @@ const CostOptimizationView: React.FC<CostOptimizationViewProps> = ({ accessToken
         <PaginationStatusAlerts
           isFetchingMore={activity.isFetchingMore}
           cancelled={activity.cancelled}
+          failed={activity.failed}
           progress={activity.progress}
           cancel={activity.cancel}
         />
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx
index 2c602033171..66db347e70f 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/PromptCachingTab.test.tsx
@@ -35,6 +35,7 @@ describe("PromptCachingTab", () => {
       isFetchingMore: false,
       progress: { currentPage: 1, totalPages: 1 },
       cancelled: false,
+      failed: false,
       cancel: vi.fn(),
     };
     render(<PromptCachingTab accessToken="test-token" activity={activity} />);
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx
index f85a667a074..c62208aacc5 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/UsageTab.test.tsx
@@ -123,6 +123,7 @@ const renderWith = (results: DailyData[], options: RenderOptions = {}) => {
         isFetchingMore: false,
         progress: { currentPage: 1, totalPages: 1 },
         cancelled: false,
+        failed: false,
         cancel: vi.fn(),
       }}
     />,
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts
index 3435b57dbc8..9f793a68bf5 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useDailyActivityRange.ts
@@ -20,6 +20,7 @@ export interface DailyActivityRange {
   isFetchingMore: boolean;
   progress: { currentPage: number; totalPages: number };
   cancelled: boolean;
+  failed: boolean;
   cancel: () => void;
 }
 
@@ -64,7 +65,7 @@ export const useScopedDailyActivityRange = (
     args: [accessToken, startTime, endTime, userId, true, apiKey],
     enabled: !!accessToken && !!startTime && !!endTime,
   };
-  const { data, loading, isFetchingMore, progress, cancelled, cancel } =
+  const { data, loading, isFetchingMore, progress, cancelled, failed, cancel } =
     usePaginatedDailyActivity(activityQueryOptions);
 
   return {
@@ -75,6 +76,7 @@ export const useScopedDailyActivityRange = (
     isFetchingMore,
     progress,
     cancelled,
+    failed,
     cancel,
   };
 };
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx
index 6c15b3c418d..debc5602277 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx
@@ -25,6 +25,7 @@ import TeamMultiSelect from "@/components/common_components/team_multi_select";
 import UserDropdown from "@/components/common_components/UserDropdown";
 import { ActivityMetrics, processActivityData } from "@/components/activity_metrics";
 import { UsageExportHeader } from "@/components/EntityUsageExport";
+import { getExportBlockedReason } from "@/components/EntityUsageExport/exportBlockedReason";
 import type { EntityType } from "@/components/EntityUsageExport/types";
 import {
   agentDailyActivityCall,
@@ -145,9 +146,11 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
 
   const {
     data: spendDataRaw,
+    loading,
     isFetchingMore,
     progress,
     cancelled,
+    failed,
     cancel,
   } = usePaginatedDailyActivity({
     fetchFn,
@@ -163,6 +166,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
     isFetchingMore: agentIsFetchingMore,
     progress: agentProgress,
     cancelled: agentCancelled,
+    failed: agentFailed,
     cancel: agentCancel,
   } = usePaginatedDailyActivity({
     fetchFn: agentDailyActivityCall,
@@ -660,11 +664,14 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
     { key: "endpoints", label: "Endpoint Activity", content: <EndpointUsage userSpendData={spendData} /> },
   ];
 
+  const spendFetchState = { loading, isFetchingMore, cancelled, failed };
+
   return (
     <div style={{ width: "100%" }} className="relative">
       <PaginationStatusAlerts
         isFetchingMore={isFetchingMore}
         cancelled={cancelled}
+        failed={failed}
         progress={progress}
         cancel={cancel}
       />
@@ -672,6 +679,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
         <PaginationStatusAlerts
           isFetchingMore={agentIsFetchingMore}
           cancelled={agentCancelled}
+          failed={agentFailed}
           progress={agentProgress}
           cancel={agentCancel}
           subject="agent data"
@@ -689,6 +697,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
         onFiltersChange={setSelectedTags}
         filterOptions={getAllTags() || undefined}
         teams={teams || []}
+        exportBlockedReason={getExportBlockedReason(spendFetchState)}
       />
       <Tabs defaultValue={tabs[0].key}>
         <TabsList className="mt-1">
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
index de353948db9..31c9288c911 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
@@ -30,6 +30,7 @@ import { ActivityMetrics, processActivityData } from "@/components/activity_metr
 import CloudZeroExportModal from "@/components/cloudzero_export_modal";
 import UserDropdown from "@/components/common_components/UserDropdown";
 import EntityUsageExportModal from "@/components/EntityUsageExport";
+import { getExportBlockedReason } from "@/components/EntityUsageExport/exportBlockedReason";
 import KeyActivityPanel from "@/components/UsagePage/components/KeyActivityPanel";
 import { Team } from "@/components/key_team_helpers/key_list";
 import {
@@ -249,6 +250,14 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
 
   const loading = aggregatedLoading || paginatedResult.loading;
 
+  const spendFetchState = {
+    loading,
+    isFetchingMore: paginatedResult.isFetchingMore,
+    cancelled: paginatedResult.cancelled,
+    failed: paginatedResult.failed,
+  };
+  const exportBlockedReason = getExportBlockedReason(spendFetchState);
+
   // Clear isDateChanging when paginated data starts arriving
   useEffect(() => {
     if (aggregatedFailed && !paginatedResult.loading && paginatedResult.data.results.length > 0) {
@@ -489,6 +498,7 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
           <PaginationStatusAlerts
             isFetchingMore={paginatedResult.isFetchingMore}
             cancelled={paginatedResult.cancelled}
+            failed={paginatedResult.failed}
             progress={paginatedResult.progress}
             cancel={paginatedResult.cancel}
           />
@@ -525,10 +535,16 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
                       <Sparkles />
                       Ask AI
                     </Button>
-                    <Button variant="outline" onClick={() => setIsGlobalExportModalOpen(true)}>
-                      <Download />
-                      Export Data
-                    </Button>
+                    <span title={exportBlockedReason}>
+                      <Button
+                        variant="outline"
+                        disabled={exportBlockedReason !== undefined}
+                        onClick={() => setIsGlobalExportModalOpen(true)}
+                      >
+                        <Download />
+                        Export Data
+                      </Button>
+                    </span>
                   </div>
                 </div>
                 {/* Cost Panel */}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts
index 0537f469920..5257e5fc8a6 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts
@@ -156,3 +156,69 @@ describe("usePaginatedDailyActivity page accumulation", () => {
     expect(result.current.data.metadata.total_spend).toBe(5.5);
   });
 });
+
+describe("usePaginatedDailyActivity failure reporting", () => {
+  const firstPage = { results: [dayOf("2026-08-16", 2)], metadata: { total_pages: 3, page: 1, total_spend: 2 } };
+  const start = new Date("2026-08-10");
+  const end = new Date("2026-08-17");
+
+  it("reports a failed range so partial totals cannot pass as the whole range", async () => {
+    const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+    const fetchFn = vi.fn((_token: string, _start: Date, _end: Date, page: number) =>
+      page === 1 ? Promise.resolve(firstPage) : Promise.reject(new Error("page 2 never came back")),
+    );
+
+    const { result } = renderHook(() =>
+      usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
+    );
+
+    await waitFor(() => expect(result.current.failed).toBe(true), { timeout: 5000 });
+
+    expect(result.current.isFetchingMore).toBe(false);
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.metadata.total_spend).toBe(2);
+    consoleError.mockRestore();
+  });
+
+  it("stays unfailed when every page arrives", async () => {
+    const pages = [
+      firstPage,
+      { results: [dayOf("2026-08-15", 1)], metadata: { total_pages: 2, page: 2, total_spend: 1 } },
+    ];
+    const fetchFn = vi.fn((_token: string, _start: Date, _end: Date, page: number) =>
+      Promise.resolve({ ...pages[page - 1], metadata: { ...pages[page - 1].metadata, total_pages: 2 } }),
+    );
+
+    const { result } = renderHook(() =>
+      usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
+    );
+
+    await waitFor(() => expect(result.current.data.metadata.page).toBe(2), { timeout: 5000 });
+
+    expect(result.current.failed).toBe(false);
+  });
+
+  it("clears the failure when a new range is requested, so the banner cannot outlive it", async () => {
+    const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+    const fetchFn = vi.fn((...callArgs: unknown[]) => {
+      const [, , , page, filter] = callArgs as [string, Date, Date, number, string | null];
+      if (filter !== "broken")
+        return Promise.resolve({ ...firstPage, metadata: { ...firstPage.metadata, total_pages: 1 } });
+      if (page === 1) return Promise.resolve({ ...firstPage, metadata: { ...firstPage.metadata, total_pages: 2 } });
+      return Promise.reject(new Error("page 2 never came back"));
+    });
+
+    const { result, rerender } = renderHook(
+      ({ filter }: { filter: string | null }) =>
+        usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, filter], enabled: true }),
+      { initialProps: { filter: "broken" as string | null } },
+    );
+
+    await waitFor(() => expect(result.current.failed).toBe(true), { timeout: 5000 });
+
+    rerender({ filter: "healthy" });
+
+    await waitFor(() => expect(result.current.failed).toBe(false), { timeout: 5000 });
+    consoleError.mockRestore();
+  });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts
index e023feda2e3..aef0021cc3d 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts
@@ -61,6 +61,8 @@ interface UsePaginatedDailyActivityReturn {
   isFetchingMore: boolean;
   progress: PaginationProgress;
   cancelled: boolean;
+  /** A page request threw, so `data` covers only part of the requested range. */
+  failed: boolean;
   cancel: () => void;
 }
 
@@ -200,6 +202,7 @@ export function usePaginatedDailyActivity({
     totalPages: 0,
   });
   const [cancelled, setCancelled] = useState(false);
+  const [failed, setFailed] = useState(false);
 
   const fetchIdRef = useRef(0);
   const cancelledRef = useRef(false);
@@ -230,12 +233,14 @@ export function usePaginatedDailyActivity({
       setIsFetchingMore(false);
       setProgress({ currentPage: 0, totalPages: 0 });
       setCancelled(false);
+      setFailed(false);
       return;
     }
 
     const currentFetchId = ++fetchIdRef.current;
     cancelledRef.current = false;
     setCancelled(false);
+    setFailed(false);
 
     const isStale = () => fetchIdRef.current !== currentFetchId || cancelledRef.current;
 
@@ -333,6 +338,7 @@ export function usePaginatedDailyActivity({
           console.error("Error fetching daily activity:", error);
           setLoading(false);
           setIsFetchingMore(false);
+          setFailed(true);
         }
       }
     };
@@ -350,5 +356,5 @@ export function usePaginatedDailyActivity({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [enabled, fetchFn, aggregatedFetchFn, argsKey]);
 
-  return { data, loading, isFetchingMore, progress, cancelled, cancel };
+  return { data, loading, isFetchingMore, progress, cancelled, failed, cancel };
 }
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx
index 52fc7605d90..460646dc39c 100644
--- a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx
+++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.test.tsx
@@ -41,6 +41,27 @@ describe("UsageExportHeader", () => {
     expect(screen.getByTestId("export-modal")).toBeInTheDocument();
   });
 
+  it("blocks the export while the data on screen does not cover the range", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(
+      <UsageExportHeader
+        {...defaultProps}
+        exportBlockedReason="Spend data is still loading, so an export would under-report. Wait for it to finish."
+      />,
+    );
+
+    const exportButton = screen.getByRole("button", { name: /export data/i });
+    expect(exportButton).toBeDisabled();
+    await user.click(exportButton);
+    expect(screen.queryByTestId("export-modal")).not.toBeInTheDocument();
+  });
+
+  it("explains why the export is blocked on hover", () => {
+    renderWithProviders(<UsageExportHeader {...defaultProps} exportBlockedReason="Spend data is still loading" />);
+
+    expect(screen.getByTitle("Spend data is still loading")).toBeInTheDocument();
+  });
+
   it("should close the export modal when onClose is called", async () => {
     const user = userEvent.setup();
     renderWithProviders(<UsageExportHeader {...defaultProps} />);
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx
index f5bb56265ed..3c4c7f6f4ee 100644
--- a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx
+++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx
@@ -34,6 +34,8 @@ interface UsageExportHeaderProps {
   customTitle?: string;
   compactLayout?: boolean;
   teams?: Team[];
+  /** Set to block the export and explain why; see getExportBlockedReason. */
+  exportBlockedReason?: string;
 }
 
 const UsageExportHeader: React.FC<UsageExportHeaderProps> = ({
@@ -50,6 +52,7 @@ const UsageExportHeader: React.FC<UsageExportHeaderProps> = ({
   customTitle,
   compactLayout = false,
   teams = [],
+  exportBlockedReason,
 }) => {
   const anchor = useComboboxAnchor();
   const [isExportModalOpen, setIsExportModalOpen] = useState(false);
@@ -121,10 +124,12 @@ const UsageExportHeader: React.FC<UsageExportHeaderProps> = ({
           )}
 
           <div className="justify-self-end">
-            <Button onClick={() => setIsExportModalOpen(true)}>
-              <Download />
-              Export Data
-            </Button>
+            <span title={exportBlockedReason}>
+              <Button disabled={exportBlockedReason !== undefined} onClick={() => setIsExportModalOpen(true)}>
+                <Download />
+                Export Data
+              </Button>
+            </span>
           </div>
         </div>
       </div>
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts
new file mode 100644
index 00000000000..0b32aef52fa
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts
@@ -0,0 +1,39 @@
+import { describe, expect, it } from "vitest";
+
+import { getExportBlockedReason, type UsageFetchState } from "./exportBlockedReason";
+
+const state = (overrides: Partial<UsageFetchState> = {}): UsageFetchState => ({
+  loading: false,
+  isFetchingMore: false,
+  cancelled: false,
+  failed: false,
+  ...overrides,
+});
+
+describe("getExportBlockedReason", () => {
+  it("lets the export through once the range has fully loaded", () => {
+    expect(getExportBlockedReason(state())).toBeUndefined();
+  });
+
+  it("blocks the first load, before any page has arrived", () => {
+    expect(getExportBlockedReason(state({ loading: true }))).toMatch(/still loading/i);
+  });
+
+  it("blocks while later pages are still arriving, which is when a CSV silently under-reports", () => {
+    expect(getExportBlockedReason(state({ isFetchingMore: true }))).toMatch(/still loading/i);
+  });
+
+  it("blocks after a stopped fetch and says a reload is what fixes it", () => {
+    const reason = getExportBlockedReason(state({ cancelled: true }));
+
+    expect(reason).toMatch(/stopped/i);
+    expect(reason).toMatch(/reload/i);
+  });
+
+  it("blocks after a failed page and names the failure rather than the stop", () => {
+    const reason = getExportBlockedReason(state({ failed: true, cancelled: true }));
+
+    expect(reason).toMatch(/failed to load/i);
+    expect(reason).not.toMatch(/stopped/i);
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts
new file mode 100644
index 00000000000..ebe67f1ccfe
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts
@@ -0,0 +1,21 @@
+export interface UsageFetchState {
+  loading: boolean;
+  isFetchingMore: boolean;
+  cancelled: boolean;
+  failed: boolean;
+}
+
+/** Why exporting what is on screen would under-report, or undefined once it covers the whole range. */
+export const getExportBlockedReason = ({
+  loading,
+  isFetchingMore,
+  cancelled,
+  failed,
+}: UsageFetchState): string | undefined => {
+  if (failed) return "Some spend data failed to load, so an export would under-report. Reload the page to try again.";
+  if (cancelled)
+    return "Loading was stopped before the whole range arrived, so an export would under-report. Reload the page to load it all.";
+  if (loading || isFetchingMore)
+    return "Spend data is still loading, so an export would under-report. Wait for it to finish.";
+  return undefined;
+};
diff --git a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx
index 5bd48b1aa4c..1eefc43ecb4 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx
@@ -33,6 +33,37 @@ describe("PaginationStatusAlerts", () => {
     expect(screen.getByText("Showing partial spend data (7/42 pages loaded)")).toBeInTheDocument();
   });
 
+  it("calls out a failed page as an error so partial totals do not read as final", () => {
+    render(
+      <PaginationStatusAlerts
+        isFetchingMore={false}
+        cancelled={false}
+        failed={true}
+        progress={{ currentPage: 7, totalPages: 42 }}
+        cancel={vi.fn()}
+      />,
+    );
+
+    expect(
+      screen.getByText(/Fetching spend data failed, so the totals below cover only part of the range \(7\/42 pages/),
+    ).toBeInTheDocument();
+  });
+
+  it("shows only the failure when a stopped fetch also failed", () => {
+    render(
+      <PaginationStatusAlerts
+        isFetchingMore={false}
+        cancelled={true}
+        failed={true}
+        progress={{ currentPage: 7, totalPages: 42 }}
+        cancel={vi.fn()}
+      />,
+    );
+
+    expect(screen.getByText(/Fetching spend data failed/)).toBeInTheDocument();
+    expect(screen.queryByText(/Showing partial spend data/)).not.toBeInTheDocument();
+  });
+
   it("names the subject it is fetching", () => {
     render(
       <PaginationStatusAlerts
diff --git a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx
index af8b8439703..ac06cfd46cf 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx
@@ -9,6 +9,7 @@ interface PaginationStatusAlertsProps {
   progress: { currentPage: number; totalPages: number };
   cancel: () => void;
   subject?: string;
+  failed?: boolean;
 }
 
 const PaginationStatusAlerts = ({
@@ -17,6 +18,7 @@ const PaginationStatusAlerts = ({
   progress,
   cancel,
   subject = "spend data",
+  failed = false,
 }: PaginationStatusAlertsProps) => (
   <>
     {isFetchingMore && (
@@ -38,7 +40,15 @@ const PaginationStatusAlerts = ({
         </AlertDescription>
       </Alert>
     )}
-    {cancelled && (
+    {failed && (
+      <Alert variant="error" className="mb-2">
+        <AlertDescription className="text-inherit">
+          Fetching {subject} failed, so the totals below cover only part of the range ({progress.currentPage}/
+          {progress.totalPages} pages loaded). Reload the page to try again.
+        </AlertDescription>
+      </Alert>
+    )}
+    {cancelled && !failed && (
       <Alert variant="info" className="mb-2">
         <AlertDescription className="text-inherit">
           Showing partial {subject} ({progress.currentPage}/{progress.totalPages} pages loaded)
diff --git a/ui/litellm-dashboard/src/components/templates/KeySavingsTab.integration.test.tsx b/ui/litellm-dashboard/src/components/templates/KeySavingsTab.integration.test.tsx
index 385c3967d02..00cd1e47e10 100644
--- a/ui/litellm-dashboard/src/components/templates/KeySavingsTab.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/templates/KeySavingsTab.integration.test.tsx
@@ -40,6 +40,7 @@ const mockActivity = (
   isFetchingMore: false,
   progress: { currentPage: 1, totalPages: 1 },
   cancelled: false,
+  failed: false,
   cancel: vi.fn(),
   ...overrides,
 });

From 868d3855abb25bcd1cb12cee68fc23f56a73d879 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:37:09 +0000
Subject: [PATCH 24/67] feat(ui): persist Models table search, filters, sort
 and page in the URL

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../components/AllModelsTab.test.tsx          | 178 +++++++++++++++---
 .../components/AllModelsTab.tsx               | 109 ++++++-----
 2 files changed, 215 insertions(+), 72 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx
index 7e47be3f5d1..7e45ff6834b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx
@@ -1,8 +1,10 @@
 import * as useAuthorizedModule from "@/app/(dashboard)/hooks/useAuthorized";
-import { fireEvent, render, screen, waitFor, within } from "@testing-library/react";
+import { fireEvent, screen, waitFor, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { OnUrlUpdateFunction } from "nuqs/adapters/testing";
+import { beforeEach, describe, expect, it, Mock, vi } from "vitest";
 
+import { renderWithProviders } from "../../../../../tests/test-utils";
 import AllModelsTab from "./AllModelsTab";
 import { STATUS_COLUMN_ID, toServerSortField } from "./ModelsTableColumns";
 
@@ -111,6 +113,9 @@ const setModelsInfo = (rows: Record<string, unknown>[], totalCount = rows.length
 
 const lastModelsInfoCall = (): ModelsInfoArgs => modelsInfoCalls[modelsInfoCalls.length - 1];
 
+const lastUrlParams = (onUrlUpdate: Mock<OnUrlUpdateFunction>): URLSearchParams | undefined =>
+  onUrlUpdate.mock.calls.at(-1)?.[0].searchParams;
+
 const SEARCH_SETTLE_MS = 400;
 
 const MOCK_AUTHORIZED = {
@@ -121,6 +126,8 @@ const MOCK_AUTHORIZED = {
   userId: "user-123",
   userEmail: "test@example.com",
   userRole: "Admin",
+  userRoleLabel: "Admin",
+  isViewOnly: false,
   premiumUser: true,
   disabledPersonalKeyCreation: false,
   showSSOBanner: false,
@@ -149,14 +156,14 @@ describe("AllModelsTab", () => {
 
   it("renders the fetched models and the server row count", async () => {
     setModelsInfo([makeRow()], 137);
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     expect(await screen.findByText("gpt-4")).toBeInTheDocument();
     expect(screen.getByTestId("pagination-range")).toHaveTextContent("Showing 1-50 of 137");
   });
 
   it("does not re-query after the mount-time debounced search settles unchanged", async () => {
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
     const callsAfterMount = modelsInfoCalls.length;
 
     await new Promise((resolve) => setTimeout(resolve, SEARCH_SETTLE_MS));
@@ -166,14 +173,14 @@ describe("AllModelsTab", () => {
 
   it("shows the empty state when the proxy returns no models", () => {
     setModelsInfo([], 0);
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     expect(screen.getByText("No models found")).toBeInTheDocument();
   });
 
   it("shows the loading skeleton while the first page is in flight", () => {
     setModelsInfo([], 0, true);
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     expect(screen.getAllByTestId("skeleton-row").length).toBeGreaterThan(0);
     expect(screen.queryByText("No models found")).not.toBeInTheDocument();
@@ -197,7 +204,7 @@ describe("AllModelsTab", () => {
 
     it.each(cases)("sorts %s using the server field %s", async (_label, columnId, serverField, firstDirection) => {
       const user = userEvent.setup();
-      render(<AllModelsTab {...defaultProps} />);
+      renderWithProviders(<AllModelsTab {...defaultProps} />);
 
       await user.click(sortHeader(columnId));
       await expectIndicator(columnId, firstDirection);
@@ -212,7 +219,7 @@ describe("AllModelsTab", () => {
 
     it("cycles a sorted column back to unsorted", async () => {
       const user = userEvent.setup();
-      render(<AllModelsTab {...defaultProps} />);
+      renderWithProviders(<AllModelsTab {...defaultProps} />);
 
       await user.click(sortHeader("model_info_updated_at"));
       await expectIndicator("model_info_updated_at", "asc");
@@ -230,7 +237,7 @@ describe("AllModelsTab", () => {
 
   it("queries the selected team and resets to the first page", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     expect(lastModelsInfoCall().teamId).toBeUndefined();
 
@@ -244,8 +251,7 @@ describe("AllModelsTab", () => {
   });
 
   it("debounces the model name search into the server query", async () => {
-    const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "claude" } });
 
@@ -254,9 +260,123 @@ describe("AllModelsTab", () => {
     });
   });
 
+  describe("URL persistence", () => {
+    it("writes the typed search to the URL and drops the page so a reload keeps the search", async () => {
+      setModelsInfo([makeRow()], 200);
+      const onUrlUpdate = vi.fn<OnUrlUpdateFunction>();
+      renderWithProviders(<AllModelsTab {...defaultProps} />, { searchParams: { page: "3" }, onUrlUpdate });
+      expect(lastModelsInfoCall().page).toBe(3);
+
+      fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "claude" } });
+
+      await waitFor(() => {
+        expect(lastUrlParams(onUrlUpdate)?.get("model_search")).toBe("claude");
+      });
+      expect(lastUrlParams(onUrlUpdate)?.get("page")).toBeNull();
+      await waitFor(() => {
+        expect(lastModelsInfoCall().page).toBe(1);
+      });
+    });
+
+    it("restores the search box and server query from ?model_search= on mount", () => {
+      renderWithProviders(<AllModelsTab {...defaultProps} />, { searchParams: { model_search: "haiku" } });
+
+      expect(screen.getByTestId("datatable-search")).toHaveValue("haiku");
+      expect(lastModelsInfoCall().search).toBe("haiku");
+    });
+
+    it("restores team, sort, page and page size from the URL into the server query", () => {
+      setModelsInfo([makeRow()], 200);
+      renderWithProviders(<AllModelsTab {...defaultProps} />, {
+        searchParams: {
+          filter_team: "team-1",
+          sort_by: "model_info_updated_at",
+          sort_order: "desc",
+          page: "2",
+          page_size: "25",
+        },
+      });
+
+      const expectedQuery: ModelsInfoArgs = {
+        teamId: "team-1",
+        sortBy: "updated_at",
+        sortOrder: "desc",
+        page: 2,
+        size: 25,
+      };
+      expect(lastModelsInfoCall()).toMatchObject(expectedQuery);
+      expect(screen.getByTestId("models-team-select")).toHaveTextContent("Engineering");
+    });
+
+    it("restores the access group and view mode from the URL", () => {
+      renderWithProviders(<AllModelsTab {...defaultProps} />, {
+        searchParams: { access_group: "sales-team", view_mode: "all" },
+      });
+
+      expect(lastModelsInfoCall().accessGroup).toBe("sales-team");
+      expect(screen.queryByText(/To access these models/)).not.toBeInTheDocument();
+    });
+
+    it("falls back to the first page and default size when the URL carries values the server rejects", () => {
+      renderWithProviders(<AllModelsTab {...defaultProps} />, { searchParams: { page: "0", page_size: "-5" } });
+
+      expect(lastModelsInfoCall().page).toBe(1);
+      expect(lastModelsInfoCall().size).toBe(50);
+    });
+
+    it("writes sort changes to the URL with the page cleared", async () => {
+      setModelsInfo([makeRow()], 200);
+      const user = userEvent.setup();
+      const onUrlUpdate = vi.fn<OnUrlUpdateFunction>();
+      renderWithProviders(<AllModelsTab {...defaultProps} />, { searchParams: { page: "2" }, onUrlUpdate });
+
+      await user.click(screen.getByTestId("sort-header-model_info_updated_at"));
+
+      await waitFor(() => {
+        expect(lastUrlParams(onUrlUpdate)?.get("sort_by")).toBe("model_info_updated_at");
+      });
+      expect(lastUrlParams(onUrlUpdate)?.get("sort_order")).toBeNull();
+      expect(lastUrlParams(onUrlUpdate)?.get("page")).toBeNull();
+
+      await user.click(screen.getByTestId("sort-header-model_info_updated_at"));
+
+      await waitFor(() => {
+        expect(lastUrlParams(onUrlUpdate)?.get("sort_order")).toBe("desc");
+      });
+    });
+
+    it("clears every table param from the URL on drawer reset", async () => {
+      setModelsInfo([makeRow()], 200);
+      const user = userEvent.setup();
+      const onUrlUpdate = vi.fn<OnUrlUpdateFunction>();
+      renderWithProviders(<AllModelsTab {...defaultProps} />, {
+        searchParams: {
+          model_search: "haiku",
+          filter_team: "team-1",
+          sort_by: "model_name",
+          page: "2",
+          view_mode: "all",
+        },
+        onUrlUpdate,
+      });
+
+      await user.click(screen.getByTestId("datatable-filters-trigger"));
+      await user.click(await screen.findByTestId("filter-drawer-reset"));
+
+      await waitFor(() => {
+        expect(lastUrlParams(onUrlUpdate)?.toString()).toBe("");
+      });
+      expect(screen.getByTestId("datatable-search")).toHaveValue("");
+      const defaultQuery: ModelsInfoArgs = { search: undefined, teamId: undefined, sortBy: undefined, page: 1 };
+      await waitFor(() => {
+        expect(lastModelsInfoCall()).toMatchObject(defaultQuery);
+      });
+    });
+  });
+
   it("applies a public model name filter through the drawer", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     await user.click(screen.getByTestId("datatable-filters-trigger"));
     await user.click(await screen.findByPlaceholderText("Filter by Public Model Name"));
@@ -270,7 +390,7 @@ describe("AllModelsTab", () => {
 
   it("renders every row the server returned for the selected model group so rows match the footer total", () => {
     setModelsInfo([makeRow(), { ...makeRow({ model_info: { id: "model-2" } }), model_name: "claude-opus" }], 2);
-    render(<AllModelsTab {...defaultProps} selectedModelGroup="claude-opus" />);
+    renderWithProviders(<AllModelsTab {...defaultProps} selectedModelGroup="claude-opus" />);
 
     const table = screen.getByRole("table");
     expect(within(table).getByText("claude-opus")).toBeInTheDocument();
@@ -280,7 +400,7 @@ describe("AllModelsTab", () => {
 
   it("asks the server for wildcard deployments instead of hiding rows client-side", () => {
     setModelsInfo([makeRow(), { ...makeRow({ model_info: { id: "model-2" } }), model_name: "openai/*" }], 2);
-    render(<AllModelsTab {...defaultProps} selectedModelGroup="wildcard" />);
+    renderWithProviders(<AllModelsTab {...defaultProps} selectedModelGroup="wildcard" />);
 
     expect(lastModelsInfoCall().wildcardOnly).toBe(true);
     expect(within(screen.getByRole("table")).getByText("gpt-4")).toBeInTheDocument();
@@ -289,7 +409,7 @@ describe("AllModelsTab", () => {
 
   it("asks the server for the selected access group instead of hiding rows client-side", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
     expect(lastModelsInfoCall().wildcardOnly).toBe(false);
 
     await user.click(screen.getByTestId("datatable-filters-trigger"));
@@ -303,20 +423,20 @@ describe("AllModelsTab", () => {
   });
 
   it("asks the server for the exact selected model group so deployments beyond the first page are found", () => {
-    render(<AllModelsTab {...defaultProps} selectedModelGroup="claude-opus" />);
+    renderWithProviders(<AllModelsTab {...defaultProps} selectedModelGroup="claude-opus" />);
 
     expect(lastModelsInfoCall().modelName).toBe("claude-opus");
     expect(lastModelsInfoCall().search).toBeUndefined();
   });
 
   it.each(["all", "wildcard"])("sends no exact model name for the %s pseudo group", (group) => {
-    render(<AllModelsTab {...defaultProps} selectedModelGroup={group} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} selectedModelGroup={group} />);
 
     expect(lastModelsInfoCall().modelName).toBeUndefined();
   });
 
   it("keeps the exact model group alongside a typed search", async () => {
-    render(<AllModelsTab {...defaultProps} selectedModelGroup="claude-opus" />);
+    renderWithProviders(<AllModelsTab {...defaultProps} selectedModelGroup="claude-opus" />);
 
     fireEvent.change(screen.getByPlaceholderText("Search model names…"), { target: { value: "opus" } });
 
@@ -326,7 +446,7 @@ describe("AllModelsTab", () => {
 
   it("resets search, filters, team and sorting from the drawer reset button", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} selectedModelGroup="gpt-4" />);
+    renderWithProviders(<AllModelsTab {...defaultProps} selectedModelGroup="gpt-4" />);
 
     await user.click(screen.getByTestId("models-team-select"));
     await user.click(await screen.findByRole("option", { name: "Engineering" }));
@@ -343,7 +463,7 @@ describe("AllModelsTab", () => {
 
   it("opens the delete modal from the row and deletes the model", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     await user.click(await screen.findByTestId("model-delete-model-1"));
     expect(await screen.findByText("Delete Model")).toBeInTheDocument();
@@ -357,7 +477,7 @@ describe("AllModelsTab", () => {
 
   it("pauses a model through the row toggle", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     await user.click(await screen.findByTestId("model-pause-toggle-model-1"));
 
@@ -368,7 +488,7 @@ describe("AllModelsTab", () => {
 
   it("opens the model settings modal from the toolbar", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     expect(screen.queryByTestId("model-settings-modal")).not.toBeInTheDocument();
     await user.click(screen.getByTestId("models-settings-trigger"));
@@ -377,7 +497,7 @@ describe("AllModelsTab", () => {
 
   it("opens the model detail view from the model ID cell", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     await user.click(await screen.findByTestId("model-id-model-1"));
 
@@ -386,7 +506,7 @@ describe("AllModelsTab", () => {
 
   it("opens the team detail view from the team ID cell", async () => {
     const user = userEvent.setup();
-    render(<AllModelsTab {...defaultProps} />);
+    renderWithProviders(<AllModelsTab {...defaultProps} />);
 
     await user.click(await screen.findByTestId("model-team-id-model-1"));
 
@@ -395,20 +515,20 @@ describe("AllModelsTab", () => {
 
   describe("virtual key hint", () => {
     it("explains personal key creation while viewing current team models", () => {
-      render(<AllModelsTab {...defaultProps} />);
+      renderWithProviders(<AllModelsTab {...defaultProps} />);
 
       expect(screen.getByText(/create a Virtual Key without selecting a team/i)).toBeInTheDocument();
     });
 
     it("links the Virtual Keys page through the migrated /ui route", () => {
-      render(<AllModelsTab {...defaultProps} />);
+      renderWithProviders(<AllModelsTab {...defaultProps} />);
 
       expect(screen.getByRole("link", { name: "Virtual Keys page" })).toHaveAttribute("href", "/ui/api-keys");
     });
 
     it("links the team hint's Virtual Keys page through the migrated /ui route", async () => {
       const user = userEvent.setup();
-      render(<AllModelsTab {...defaultProps} />);
+      renderWithProviders(<AllModelsTab {...defaultProps} />);
 
       await user.click(screen.getByTestId("models-team-select"));
       await user.click(await screen.findByRole("option", { name: "Engineering" }));
@@ -419,7 +539,7 @@ describe("AllModelsTab", () => {
 
     it("names the selected team in the hint", async () => {
       const user = userEvent.setup();
-      render(<AllModelsTab {...defaultProps} />);
+      renderWithProviders(<AllModelsTab {...defaultProps} />);
 
       await user.click(screen.getByTestId("models-team-select"));
       await user.click(await screen.findByRole("option", { name: "Engineering" }));
@@ -429,7 +549,7 @@ describe("AllModelsTab", () => {
 
     it("hides the hint when viewing all available models", async () => {
       const user = userEvent.setup();
-      render(<AllModelsTab {...defaultProps} />);
+      renderWithProviders(<AllModelsTab {...defaultProps} />);
 
       await user.click(screen.getByTestId("models-view-select"));
       await user.click(await screen.findByRole("option", { name: "All Available Models" }));
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
index ccb9f90f9a3..efe74a273d6 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
@@ -10,10 +10,11 @@ import { toast } from "@/lib/toast";
 import { uiHref } from "@/utils/uiHref";
 import { modelDeleteCall, modelPatchUpdateCall } from "@/components/networking";
 import { useQueryClient } from "@tanstack/react-query";
-import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer";
-import { ColumnFiltersState, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table";
+import { useDebouncedValue } from "@tanstack/react-pacer/debouncer";
+import { ColumnFiltersState, functionalUpdate, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table";
 import { Info } from "lucide-react";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { parseAsInteger, parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs";
+import { useCallback, useMemo, useState } from "react";
 
 import { useModelsInfo } from "../../hooks/models/useModels";
 import { transformModelData } from "../utils/modelDataTransformer";
@@ -28,7 +29,19 @@ import { ACCESS_GROUPS_COLUMN_ID, MODEL_NAME_COLUMN_ID, toServerSortField } from
 
 const SEARCH_DEBOUNCE_WAIT_MS = 200;
 const DEFAULT_PAGE_SIZE = 50;
-const DEFAULT_PAGINATION: PaginationState = { pageIndex: 0, pageSize: DEFAULT_PAGE_SIZE };
+
+const MODEL_VIEW_MODES = ["current_team", "all"] as const satisfies readonly ModelViewMode[];
+
+const TABLE_STATE = {
+  model_search: parseAsString.withDefault(""),
+  view_mode: parseAsStringLiteral(MODEL_VIEW_MODES).withDefault("current_team"),
+  filter_team: parseAsString.withDefault(PERSONAL_TEAM_VALUE),
+  access_group: parseAsString.withDefault(""),
+  sort_by: parseAsString.withDefault(""),
+  sort_order: parseAsStringLiteral(["asc", "desc"] as const).withDefault("asc"),
+  page: parseAsInteger.withDefault(1),
+  page_size: parseAsInteger.withDefault(DEFAULT_PAGE_SIZE),
+};
 
 interface AllModelsTabProps {
   selectedModelGroup: string | null;
@@ -52,34 +65,28 @@ const AllModelsTab = ({
   const { data: teams, isLoading: isLoadingTeams } = useTeams();
   const queryClient = useQueryClient();
 
-  const [modelNameSearch, setModelNameSearch] = useState<string>("");
-  const [debouncedSearch, setDebouncedSearch] = useState<string>("");
-  const [modelViewMode, setModelViewMode] = useState<ModelViewMode>("current_team");
-  const [selectedTeamValue, setSelectedTeamValue] = useState<string>(PERSONAL_TEAM_VALUE);
-  const [selectedModelAccessGroupFilter, setSelectedModelAccessGroupFilter] = useState<string | null>(null);
-  const [pagination, setPagination] = useState<PaginationState>(DEFAULT_PAGINATION);
-  const [sorting, setSorting] = useState<SortingState>([]);
+  const [tableState, setTableState] = useQueryStates(TABLE_STATE);
+  const modelNameSearch = tableState.model_search;
+  const [debouncedSearch] = useDebouncedValue(modelNameSearch, { wait: SEARCH_DEBOUNCE_WAIT_MS });
+  const modelViewMode = tableState.view_mode;
+  const selectedTeamValue = tableState.filter_team;
+  const selectedModelAccessGroupFilter = tableState.access_group || null;
+  const pagination = useMemo<PaginationState>(
+    () => ({
+      pageIndex: Math.max(tableState.page, 1) - 1,
+      pageSize: tableState.page_size >= 1 ? tableState.page_size : DEFAULT_PAGE_SIZE,
+    }),
+    [tableState.page, tableState.page_size],
+  );
+  const sorting = useMemo<SortingState>(
+    () => (tableState.sort_by ? [{ id: tableState.sort_by, desc: tableState.sort_order === "desc" }] : []),
+    [tableState.sort_by, tableState.sort_order],
+  );
   const [isModelSettingsModalVisible, setIsModelSettingsModalVisible] = useState(false);
   const [deleteModalModelId, setDeleteModalModelId] = useState<string | null>(null);
   const [deleteLoading, setDeleteLoading] = useState(false);
   const [pausingModelId, setPausingModelId] = useState<string | null>(null);
 
-  const resetToFirstPage = useCallback(() => {
-    setPagination((previous) => (previous.pageIndex === 0 ? previous : { ...previous, pageIndex: 0 }));
-  }, []);
-
-  const debouncedUpdateSearch = useDebouncedCallback(
-    (value: string) => {
-      setDebouncedSearch(value);
-      resetToFirstPage();
-    },
-    { wait: SEARCH_DEBOUNCE_WAIT_MS },
-  );
-
-  useEffect(() => {
-    debouncedUpdateSearch(modelNameSearch);
-  }, [modelNameSearch, debouncedUpdateSearch]);
-
   const teamIdForQuery = selectedTeamValue === PERSONAL_TEAM_VALUE ? undefined : selectedTeamValue;
   const isConcreteModelGroup =
     Boolean(selectedModelGroup) &&
@@ -152,33 +159,49 @@ const AllModelsTab = ({
     [selectedModelGroup, selectedModelAccessGroupFilter],
   );
 
+  const handleSearchChange = useCallback(
+    (value: string) => {
+      void setTableState({ model_search: value || null, page: null });
+    },
+    [setTableState],
+  );
+
   const handleColumnFiltersChange: OnChangeFn<ColumnFiltersState> = (updater) => {
-    const next = typeof updater === "function" ? updater(columnFilters) : updater;
+    const next = functionalUpdate(updater, columnFilters);
     const modelGroup = next.find((entry) => entry.id === MODEL_NAME_COLUMN_ID)?.value;
     const accessGroup = next.find((entry) => entry.id === ACCESS_GROUPS_COLUMN_ID)?.value;
     setSelectedModelGroup(typeof modelGroup === "string" ? modelGroup : ALL_MODEL_GROUPS_VALUE);
-    setSelectedModelAccessGroupFilter(typeof accessGroup === "string" ? accessGroup : null);
-    resetToFirstPage();
+    void setTableState({ access_group: typeof accessGroup === "string" ? accessGroup : null, page: null });
   };
 
   const handleSortingChange: OnChangeFn<SortingState> = (updater) => {
-    setSorting(typeof updater === "function" ? updater(sorting) : updater);
-    resetToFirstPage();
+    const active = functionalUpdate(updater, sorting)[0];
+    void setTableState({
+      sort_by: active?.id ?? null,
+      sort_order: active?.desc ? "desc" : null,
+      page: null,
+    });
   };
 
+  const handlePaginationChange = useCallback<OnChangeFn<PaginationState>>(
+    (updater) => {
+      const next = functionalUpdate(updater, pagination);
+      void setTableState({ page: next.pageIndex + 1, page_size: next.pageSize });
+    },
+    [pagination, setTableState],
+  );
+
   const handleTeamChange = (value: string) => {
-    setSelectedTeamValue(value);
-    resetToFirstPage();
+    void setTableState({ filter_team: value, page: null });
+  };
+
+  const handleViewModeChange = (value: ModelViewMode) => {
+    void setTableState({ view_mode: value });
   };
 
   const resetFilters = () => {
-    setModelNameSearch("");
     setSelectedModelGroup(ALL_MODEL_GROUPS_VALUE);
-    setSelectedModelAccessGroupFilter(null);
-    setSelectedTeamValue(PERSONAL_TEAM_VALUE);
-    setModelViewMode("current_team");
-    setPagination(DEFAULT_PAGINATION);
-    setSorting([]);
+    void setTableState(null);
   };
 
   const teamOptions = useMemo(
@@ -264,18 +287,18 @@ const AllModelsTab = ({
           sorting={sorting}
           onSortingChange={handleSortingChange}
           pagination={pagination}
-          onPaginationChange={setPagination}
+          onPaginationChange={handlePaginationChange}
           columnFilters={columnFilters}
           onColumnFiltersChange={handleColumnFiltersChange}
           onResetFilters={resetFilters}
           searchValue={modelNameSearch}
-          onSearchChange={setModelNameSearch}
+          onSearchChange={handleSearchChange}
           teamOptions={teamOptions}
           selectedTeamValue={selectedTeamValue}
           onTeamChange={handleTeamChange}
           isLoadingTeams={isLoadingTeams}
           viewMode={modelViewMode}
-          onViewModeChange={setModelViewMode}
+          onViewModeChange={handleViewModeChange}
           onOpenModelSettings={handleOpenModelSettings}
           availableModelGroups={availableModelGroups}
           availableModelAccessGroups={availableModelAccessGroups}

From 96bffb1290b7c53399b33a585f71ccc65e892c39 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:39:39 +0000
Subject: [PATCH 25/67] fix(passthrough): attribute Vertex passthrough
 successes to the resolved router deployment

The Vertex passthrough route resolved a router deployment only to rewrite the
upstream URL and dropped its model_info, so the standard logging payload and
the Prometheus litellm_deployment_success_responses_total counter carried
model_id="". Carry the deployment's model_info through request.state into the
passthrough logging metadata, where it overrides any client-supplied model_info.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../llm_passthrough_endpoints.py              | 20 +++--
 .../pass_through_endpoints.py                 |  8 ++
 .../pass_through_endpoints.py                 |  4 +
 .../test_pass_through_endpoints.py            | 27 +++++++
 .../test_vertex_passthrough_load_balancing.py | 77 +++++++++++++++++++
 5 files changed, 130 insertions(+), 6 deletions(-)

diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index 28a8bab1f24..3c2ae02dc52 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -77,6 +77,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
 from litellm.secret_managers.main import get_secret_str, str_to_bool
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
+    LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
     LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
 )
 from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
@@ -1322,7 +1323,7 @@ def _resolve_vertex_model_from_router(
     endpoint: str,
     vertex_project: str | None,
     vertex_location: str | None,
-) -> tuple[str, str, str | None, str | None]:
+) -> tuple[str, str, str | None, str | None, Mapping[str, object] | None]:
     """
     Resolve Vertex AI model configuration from router.
 
@@ -1335,18 +1336,21 @@ def _resolve_vertex_model_from_router(
         vertex_location: Current vertex location (may be from URL)
 
     Returns:
-        tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location)
-        with resolved values from router config
+        tuple of (encoded_endpoint, endpoint, vertex_project, vertex_location, deployment_model_info)
+        with resolved values from router config; deployment_model_info is the resolved
+        deployment's `model_info`, or None when no deployment matched
     """
     if not llm_router:
-        return encoded_endpoint, endpoint, vertex_project, vertex_location
+        return encoded_endpoint, endpoint, vertex_project, vertex_location, None
 
     try:
         deployment: Final = llm_router.get_available_deployment_for_pass_through(model=model_id)
         if not deployment:
-            return encoded_endpoint, endpoint, vertex_project, vertex_location
+            return encoded_endpoint, endpoint, vertex_project, vertex_location, None
 
         litellm_params: Final = deployment.get("litellm_params", {})
+        model_info: Final = deployment.get("model_info")
+        deployment_model_info: Final = model_info if isinstance(model_info, Mapping) else None
 
         # Always override with router config values (they take precedence over URL values)
         config_vertex_project: Final = litellm_params.get("vertex_project")
@@ -1387,10 +1391,11 @@ def _resolve_vertex_model_from_router(
                 encoded_endpoint = encoded_endpoint.replace(model_id, actual_model)
                 endpoint = endpoint.replace(model_id, actual_model)
 
+        return encoded_endpoint, endpoint, vertex_project, vertex_location, deployment_model_info
     except Exception as e:
         verbose_proxy_logger.debug("Error resolving vertex model from router for model %s: %s", model_id, e)
 
-    return encoded_endpoint, endpoint, vertex_project, vertex_location
+    return encoded_endpoint, endpoint, vertex_project, vertex_location, None
 
 
 def _is_bedrock_agent_runtime_route(endpoint: str) -> bool:
@@ -2134,6 +2139,7 @@ async def _base_vertex_proxy_route(
                 endpoint,
                 vertex_project,
                 vertex_location,
+                deployment_model_info,
             ) = _resolve_vertex_model_from_router(
                 model_id=model_id,
                 llm_router=llm_router,
@@ -2142,6 +2148,8 @@ async def _base_vertex_proxy_route(
                 vertex_project=vertex_project,
                 vertex_location=vertex_location,
             )
+            if deployment_model_info:
+                setattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, deployment_model_info)
 
     vertex_credentials: Final = passthrough_endpoint_router.get_vertex_credentials(
         project_id=vertex_project,
diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index b66c295d1aa..5d5275abad0 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -96,6 +96,7 @@ from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.custom_http import httpxSpecialProvider
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
+    LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
     LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
     LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
     EndpointType,
@@ -613,6 +614,11 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         _metadata.update(
             LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
         )
+        deployment_model_info: Final = getattr(
+            getattr(request, "state", None), LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, None
+        )
+        if isinstance(deployment_model_info, Mapping):
+            _metadata["model_info"] = dict(deployment_model_info)
 
         kwargs: Final = {
             "litellm_params": {
@@ -2002,6 +2008,8 @@ def create_pass_through_route(
                         delattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY)
                     if hasattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY):
                         delattr(request.state, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY)
+                    if hasattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY):
+                        delattr(request.state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY)
 
             # The upstream withholds its response headers until its first token, so
             # the whole time-to-first-token is spent inside _relay with nothing on
diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py
index b5ebcafb9f0..fb12daab199 100644
--- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py
+++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py
@@ -11,6 +11,10 @@ LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY: Final = "litellm_pass_through_custom
 # exact byte/string body, such as AWS SigV4-signed requests.
 LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY: Final = "litellm_pass_through_raw_body"
 
+# Request.state key carrying the `model_info` of the router deployment a provider
+# route resolved (e.g. Vertex), so logging attributes the call to that deployment.
+LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY: Final = "litellm_pass_through_deployment_model_info"
+
 # Attribute set on the FastAPI endpoint function of every user-defined pass-through
 # route. Auth reads it off the dispatched endpoint (``request.scope["endpoint"]``) to
 # decide whether a request body ``model`` names an upstream model rather than a
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
index 11066d4ed38..126c4ae54f0 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
@@ -34,6 +34,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
+    LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
     LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
 )
 from litellm.proxy.pass_through_endpoints.success_handler import (
@@ -5934,6 +5935,32 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key
     )
 
 
+@pytest.mark.parametrize("client_metadata_key", ["litellm_metadata", "metadata"])
+def test_passthrough_logs_the_resolved_deployment_model_info_over_the_request_body(client_metadata_key: str):
+    """A provider route that resolved a router deployment stashes its model_info on request.state. That
+    deployment, not a model_info the client put in its own body, is what spend logs and metrics attribute
+    the call to (LIT-1761: passthrough successes carried model_id="")."""
+    mock_request = MagicMock(spec=Request)
+    mock_request.method = "POST"
+    mock_request.url = "http://0.0.0.0:4000/vertex_ai/v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent"
+    mock_request.headers = Headers({})
+    mock_request.scope = {}
+    mock_request.state = SimpleNamespace(
+        **{LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY: {"id": "vertex-gemini-38-flash-dep"}}
+    )
+
+    kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
+        request=mock_request,
+        user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
+        passthrough_logging_payload=MagicMock(),
+        logging_obj=MagicMock(),
+        _parsed_body={client_metadata_key: {"model_info": {"id": "client-forged-id"}}},
+        litellm_call_id="lit-1761-call-id",
+    )
+
+    assert kwargs["litellm_params"]["metadata"]["model_info"] == {"id": "vertex-gemini-38-flash-dep"}
+
+
 @pytest.mark.asyncio
 async def test_chat_completion_pass_through_endpoint_answers_an_openai_typed_error_for_an_unknown_model(
     monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
index e8fd5579631..dde47004adc 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
@@ -1,6 +1,8 @@
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
+from fastapi import Request
+from starlette.datastructures import Headers, State
 
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
@@ -8,6 +10,9 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     _base_vertex_proxy_route,
     _upstream_headers_for_vertex_route,
 )
+from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+    HttpPassThroughEndpointHelpers,
+)
 from litellm.types.router import DeploymentTypedDict
 
 
@@ -758,3 +763,75 @@ async def test_vertex_passthrough_custom_model_name_replaced_in_url():
         assert (
             "gemini-3-pro" in target_url
         ), f"Actual Vertex AI model name should be in target URL. Got: {target_url}"
+
+
+@pytest.mark.asyncio
+async def test_vertex_passthrough_attributes_the_call_to_the_resolved_deployment():
+    """The router deployment that rewrote the upstream URL is the one the logging kwargs must name, so
+    the Prometheus model_id label (and SpendLogs.model_id) on a Vertex passthrough success reads the
+    deployment's id instead of "" (LIT-1761)."""
+    mock_request = MagicMock(spec=Request)
+    mock_request.method = "POST"
+    mock_request.url = "http://0.0.0.0:4000/vertex_ai/v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent"
+    mock_request.headers = Headers({})
+    mock_request.scope = {}
+    mock_request.state = State()
+    mock_handler = MagicMock()
+    mock_handler.get_default_base_target_url.return_value = "https://aiplatform.googleapis.com"
+
+    mock_router = MagicMock()
+    mock_router.get_available_deployment_for_pass_through.return_value = {
+        "model_name": "gemini-3.8-flash",
+        "litellm_params": {
+            "model": "vertex_ai/gemini-3.8-flash",
+            "vertex_project": "p",
+            "vertex_location": "global",
+            "use_in_pass_through": True,
+        },
+        "model_info": {"id": "vertex-gemini-38-flash-dep"},
+    }
+
+    async def relay_returning_logging_kwargs(
+        request: Request, fastapi_response: object, user_api_key_dict: UserAPIKeyAuth
+    ) -> dict:
+        return HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
+            request=request,
+            user_api_key_dict=user_api_key_dict,
+            passthrough_logging_payload=MagicMock(),
+            logging_obj=MagicMock(),
+            _parsed_body={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]},
+            litellm_call_id="lit-1761-call-id",
+        )
+
+    with (
+        patch(  # test-quality-ok: the route reads this proxy global at call time, nothing injects it
+            "litellm.proxy.proxy_server.llm_router", mock_router
+        ),
+        patch(  # test-quality-ok: the route reads this proxy global at call time, nothing injects it
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router"
+        ) as mock_pt_router,
+        patch(  # test-quality-ok: the route offers no injection point for its header preparation
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._prepare_vertex_auth_headers",
+            new_callable=AsyncMock,
+            return_value=({}, False, "p", "global"),
+        ),
+        patch(  # test-quality-ok: the relay is captured here to read the logging kwargs, the route offers no seam
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
+            return_value=relay_returning_logging_kwargs,
+        ),
+        patch(  # test-quality-ok: the route calls auth directly rather than through Depends
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.user_api_key_auth",
+            new_callable=AsyncMock,
+            return_value=UserAPIKeyAuth(api_key="hashed-key"),
+        ),
+    ):
+        mock_pt_router.get_vertex_credentials.return_value = MagicMock()
+
+        logging_kwargs = await _base_vertex_proxy_route(
+            endpoint="v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent",
+            request=mock_request,
+            fastapi_response=MagicMock(),
+            get_vertex_pass_through_handler=mock_handler,
+        )
+
+    assert logging_kwargs["litellm_params"]["metadata"]["model_info"]["id"] == "vertex-gemini-38-flash-dep"

From 51a4cb9fdd99be0c41332a57cc3ca08f210efc3d Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:48:06 +0000
Subject: [PATCH 26/67] fix(proxy): key model rpm/tpm override takes precedence
 over team model limit

A key inside a team with model_rpm_limit / model_tpm_limit in team metadata could not
override those limits for itself: the v3 limiter always added the team's per-model
descriptor next to the key's, so the tighter team limit won. The docs already say the
resolution order is key metadata > key model_max_budget > team metadata

get_key_own_model_rate_limit returns only what the key sets on itself, and the team
descriptor now carries only the metrics the key does not override, so an rpm-only
override still leaves the team tpm pool enforced

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/auth_utils.py              | 60 ++++++++--------
 .../hooks/parallel_request_limiter_v3.py      | 62 ++++++++--------
 .../proxy/auth/test_auth_utils.py             | 30 ++++++++
 .../hooks/test_parallel_request_limiter_v3.py | 70 ++++++++++++++++++-
 4 files changed, 165 insertions(+), 57 deletions(-)

diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index dc304a156cf..5ff85ad4a0e 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -976,6 +976,32 @@ def _get_deployment_default_tpm_limit(model_name: str) -> int | None:
     return _get_deployment_default_limit(model_name, "default_api_key_tpm_limit")
 
 
+def get_key_own_model_rate_limit(
+    user_api_key_dict: UserAPIKeyAuth,
+    rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
+) -> dict[str, int] | None:
+    """
+    Per-model limit the key sets on itself: key metadata first, then model_max_budget.
+
+    Unlike get_key_model_rpm_limit / get_key_model_tpm_limit this never falls back to the
+    team, so callers can tell a key override apart from an inherited team limit.
+    """
+    if user_api_key_dict.metadata:
+        result: Final = user_api_key_dict.metadata.get(rate_limit_key)
+        if result:
+            return result
+
+    if not user_api_key_dict.model_max_budget:
+        return None
+    budget_key: Final = "rpm_limit" if rate_limit_key == "model_rpm_limit" else "tpm_limit"
+    model_limit: Final = {
+        model: budget[budget_key]
+        for model, budget in user_api_key_dict.model_max_budget.items()
+        if isinstance(budget, dict) and budget.get(budget_key) is not None
+    }
+    return model_limit or None
+
+
 def get_key_model_rpm_limit(
     user_api_key_dict: UserAPIKeyAuth,
     model_name: str | None = None,
@@ -989,20 +1015,9 @@ def get_key_model_rpm_limit(
     3. Team metadata (model_rpm_limit)
     4. Deployment default_api_key_rpm_limit (when model_name is provided)
     """
-    # 1. Check key metadata first (takes priority)
-    if user_api_key_dict.metadata:
-        result: Final = user_api_key_dict.metadata.get("model_rpm_limit")
-        if result:
-            return result
-
-    # 2. Check model_max_budget
-    if user_api_key_dict.model_max_budget:
-        model_rpm_limit: Final[dict[str, int]] = {}
-        for model, budget in user_api_key_dict.model_max_budget.items():
-            if isinstance(budget, dict) and budget.get("rpm_limit") is not None:
-                model_rpm_limit[model] = budget["rpm_limit"]
-        if model_rpm_limit:
-            return model_rpm_limit
+    key_own_limit: Final = get_key_own_model_rate_limit(user_api_key_dict, "model_rpm_limit")
+    if key_own_limit is not None:
+        return key_own_limit
 
     # 3. Fallback to team metadata
     if user_api_key_dict.team_metadata:
@@ -1032,20 +1047,9 @@ def get_key_model_tpm_limit(
     3. Team metadata (model_tpm_limit)
     4. Deployment default_api_key_tpm_limit (when model_name is provided)
     """
-    # 1. Check key metadata first (takes priority)
-    if user_api_key_dict.metadata:
-        result: Final = user_api_key_dict.metadata.get("model_tpm_limit")
-        if result:
-            return result
-
-    # 2. Check model_max_budget (iterate per-model like RPM does)
-    if user_api_key_dict.model_max_budget:
-        model_tpm_limit: Final[dict[str, int]] = {}
-        for model, budget in user_api_key_dict.model_max_budget.items():
-            if isinstance(budget, dict) and budget.get("tpm_limit") is not None:
-                model_tpm_limit[model] = budget["tpm_limit"]
-        if model_tpm_limit:
-            return model_tpm_limit
+    key_own_limit: Final = get_key_own_model_rate_limit(user_api_key_dict, "model_tpm_limit")
+    if key_own_limit is not None:
+        return key_own_limit
 
     # 3. Fallback to team metadata
     if user_api_key_dict.team_metadata:
diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py
index 8ca4124521a..87336830976 100644
--- a/litellm/proxy/hooks/parallel_request_limiter_v3.py
+++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py
@@ -41,6 +41,7 @@ from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.auth_utils import (
     ESTIMATED_OUTPUT_TOKENS_FIELD,
     get_estimated_output_tokens,
+    get_key_own_model_rate_limit,
     get_key_tag_rpm_limit,
     get_model_rate_limit_from_metadata,
 )
@@ -2892,41 +2893,46 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             return batch_limiter
         return None
 
+    def _inherited_team_model_limit(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        requested_model: str,
+        rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
+    ) -> int | None:
+        """Team per-model limit this key inherits: None when the key sets its own limit for the model."""
+        team_limits: Final = get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", rate_limit_key)
+        team_limit: Final = team_limits.get(requested_model) if team_limits else None
+        if team_limit is None:
+            return None
+        key_own_limits: Final = get_key_own_model_rate_limit(user_api_key_dict, rate_limit_key)
+        if key_own_limits and key_own_limits.get(requested_model) is not None:
+            return None
+        return team_limit
+
     def _add_team_model_rate_limit_descriptor_from_metadata(
         self,
         user_api_key_dict: UserAPIKeyAuth,
         requested_model: str | None,
         descriptors: list[RateLimitDescriptor],
     ) -> None:
-        """Add team model rate limit descriptor from team_metadata if applicable."""
-        if (
-            get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", "model_rpm_limit") is not None
-            or get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", "model_tpm_limit") is not None
-        ):
-            _tpm_limit_for_team_model: Final = (
-                get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", "model_tpm_limit") or {}
+        """Add the team's per-model descriptor for the metrics the key does not override itself."""
+        if requested_model is None:
+            return
+        team_rpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, requested_model, "model_rpm_limit")
+        team_tpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, requested_model, "model_tpm_limit")
+        if team_rpm_limit is None and team_tpm_limit is None:
+            return
+        descriptors.append(
+            RateLimitDescriptor(
+                key="model_per_team",
+                value=f"{user_api_key_dict.team_id}:{requested_model}",
+                rate_limit={
+                    "requests_per_unit": team_rpm_limit,
+                    "tokens_per_unit": team_tpm_limit,
+                    "window_size": self.window_size,
+                },
             )
-            _rpm_limit_for_team_model: Final = (
-                get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", "model_rpm_limit") or {}
-            )
-            should_check_rate_limit: Final = (
-                requested_model in _tpm_limit_for_team_model or requested_model in _rpm_limit_for_team_model
-            )
-
-            if should_check_rate_limit and requested_model is not None:
-                model_specific_tpm_limit: Final = _tpm_limit_for_team_model.get(requested_model)
-                model_specific_rpm_limit: Final = _rpm_limit_for_team_model.get(requested_model)
-                descriptors.append(
-                    RateLimitDescriptor(
-                        key="model_per_team",
-                        value=f"{user_api_key_dict.team_id}:{requested_model}",
-                        rate_limit={
-                            "requests_per_unit": model_specific_rpm_limit,
-                            "tokens_per_unit": model_specific_tpm_limit,
-                            "window_size": self.window_size,
-                        },
-                    )
-                )
+        )
 
     def _add_project_model_rate_limit_descriptor_from_metadata(
         self,
diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py
index bd6a14cad21..639c697726e 100644
--- a/tests/test_litellm/proxy/auth/test_auth_utils.py
+++ b/tests/test_litellm/proxy/auth/test_auth_utils.py
@@ -22,6 +22,7 @@ from litellm.proxy.auth.auth_utils import (
     get_key_mcp_rpm_limit,
     get_key_model_rpm_limit,
     get_key_model_tpm_limit,
+    get_key_own_model_rate_limit,
     get_key_tag_rpm_limit,
     get_model_from_request,
     get_project_model_rpm_limit,
@@ -141,6 +142,35 @@ class TestLogOnceIfBudgetReservationDisabled:
 class TestGetKeyModelRpmLimit:
     """Tests for get_key_model_rpm_limit function."""
 
+    def test_own_limit_excludes_team_metadata(self):
+        """A team-only limit is inherited, not owned: the key resolves it but does not override it."""
+        user_api_key_dict = UserAPIKeyAuth(
+            api_key="sk-123",
+            metadata={"some_other_key": "value"},
+            team_metadata={"model_rpm_limit": {"gpt-4": 50}, "model_tpm_limit": {"gpt-4": 500}},
+        )
+        assert get_key_model_rpm_limit(user_api_key_dict) == {"gpt-4": 50}
+        assert get_key_own_model_rate_limit(user_api_key_dict, "model_rpm_limit") is None
+        assert get_key_own_model_rate_limit(user_api_key_dict, "model_tpm_limit") is None
+
+    def test_own_limit_resolves_metadata_then_model_max_budget(self):
+        from_metadata = UserAPIKeyAuth(
+            api_key="sk-123",
+            metadata={"model_rpm_limit": {"gpt-4": 100}},
+            model_max_budget={"gpt-4": {"rpm_limit": 10, "tpm_limit": 1000}},
+            team_metadata={"model_rpm_limit": {"gpt-4": 50}},
+        )
+        assert get_key_own_model_rate_limit(from_metadata, "model_rpm_limit") == {"gpt-4": 100}
+        assert get_key_own_model_rate_limit(from_metadata, "model_tpm_limit") == {"gpt-4": 1000}
+
+        from_budget = UserAPIKeyAuth(
+            api_key="sk-123",
+            model_max_budget={"gpt-4": {"rpm_limit": 10}, "gpt-3.5-turbo": {"tpm_limit": 1000}},
+            team_metadata={"model_rpm_limit": {"gpt-4": 50}},
+        )
+        assert get_key_own_model_rate_limit(from_budget, "model_rpm_limit") == {"gpt-4": 10}
+        assert get_key_own_model_rate_limit(from_budget, "model_tpm_limit") == {"gpt-3.5-turbo": 1000}
+
     def test_returns_key_metadata_when_present(self):
         """Key metadata takes priority over team metadata."""
         user_api_key_dict = UserAPIKeyAuth(
diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
index 48f980086fd..c2b88776dc5 100644
--- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
+++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
@@ -6311,7 +6311,7 @@ async def test_an_open_circuit_breaker_reads_the_sliding_window_locally_without_
         (
             {
                 "team_id": "t",
-                "metadata": {"model_rpm_limit": {"test-model": 100}},
+                "metadata": {"model_rpm_limit": {"other-model": 100}},
                 "team_metadata": {"model_rpm_limit": {"test-model": 1}},
             },
             {},
@@ -6529,3 +6529,71 @@ async def test_request_capacity_rejection_keeps_existing_redis_mirror():
                 pytest.fail("rejection released another request's mirrored slot")
         assert exc.value.status_code == 429
         assert await cache.async_get_cache(counter_key, local_only=True) == 1
+
+
+@pytest.mark.parametrize(
+    "key_limits",
+    [
+        {"metadata": {"model_rpm_limit": {"test-model": 3}}},
+        {"model_max_budget": {"test-model": {"rpm_limit": 3}}},
+    ],
+)
+@pytest.mark.asyncio
+async def test_key_model_rpm_override_takes_precedence_over_team_model_rpm_limit(key_limits):
+    cache = DualCache()
+    handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache))
+    auth = UserAPIKeyAuth(
+        api_key=hash_token("sk-key-override"),
+        team_id="t",
+        team_metadata={"model_rpm_limit": {"test-model": 1}},
+        **key_limits,
+    )
+
+    async def request():
+        await handler.async_pre_call_hook(
+            user_api_key_dict=auth, cache=cache, data={"model": "test-model"}, call_type="acompletion"
+        )
+
+    for _ in range(3):
+        await request()
+    with pytest.raises(HTTPException) as exc:
+        await request()
+    assert exc.value.status_code == 429
+    assert "model_per_key" in str(exc.value.detail)
+
+
+@pytest.mark.parametrize(
+    "key_limits, override_key_gets_through",
+    [
+        ({"model_rpm_limit": {"test-model": 10}}, False),
+        ({"model_rpm_limit": {"test-model": 10}, "model_tpm_limit": {"test-model": 5000}}, True),
+    ],
+    ids=["rpm_only_override_still_shares_team_tpm", "rpm_and_tpm_override_leaves_team_tpm"],
+)
+@pytest.mark.asyncio
+async def test_key_model_rpm_override_keeps_team_model_tpm_limit(key_limits, override_key_gets_through):
+    cache = DualCache()
+    handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache))
+    team_metadata = {"model_rpm_limit": {"test-model": 5}, "model_tpm_limit": {"test-model": 500}}
+    sibling_key = UserAPIKeyAuth(api_key=hash_token("sk-sibling"), team_id="t", team_metadata=team_metadata)
+    override_key = UserAPIKeyAuth(
+        api_key=hash_token("sk-key-override"), team_id="t", metadata=key_limits, team_metadata=team_metadata
+    )
+
+    async def request(auth):
+        await handler.async_pre_call_hook(
+            user_api_key_dict=auth,
+            cache=cache,
+            data={"model": "test-model", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 300},
+            call_type="acompletion",
+        )
+
+    await request(sibling_key)
+    if override_key_gets_through:
+        await request(override_key)
+        return
+    with pytest.raises(HTTPException) as exc:
+        await request(override_key)
+    assert exc.value.status_code == 429
+    assert "model_per_team" in str(exc.value.detail)
+    assert exc.value.headers["rate_limit_type"] == "tokens"

From 77d913958d660a2a1dca8639c109cd24521e2393 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:53:29 +0000
Subject: [PATCH 27/67] feat(openai): add openai_system_messages_first to put
 system messages first for prompt caching

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/__init__.py                           |  1 +
 litellm/constants.py                          |  2 +
 .../prompt_templates/common_utils.py          | 16 ++++
 litellm/llms/azure/chat/gpt_transformation.py |  4 +-
 .../llms/openai/chat/gpt_transformation.py    | 19 ++++-
 litellm/proxy/proxy_server.py                 | 10 +++
 ...ore_utils_prompt_templates_common_utils.py | 33 +++++++++
 .../test_azure_chat_gpt_transformation.py     | 26 +++++++
 ...test_azure_chat_o_series_transformation.py | 21 ++++++
 .../chat/test_openai_gpt_transformation.py    | 74 +++++++++++++++++++
 tests/test_litellm/proxy/test_proxy_server.py | 11 +++
 .../general_settings.integration.test.tsx     | 42 +++++++++++
 .../_components/general_settings.tsx          | 22 +++++-
 13 files changed, 276 insertions(+), 5 deletions(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index ccfbf80369f..55e258a2c27 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -343,6 +343,7 @@ _anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_
 anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
     "1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
 )
+openai_system_messages_first: bool = os.getenv("LITELLM_OPENAI_SYSTEM_MESSAGES_FIRST", "false").lower() == "true"
 disable_vertex_batch_output_transformation: bool = False
 extra_spend_tag_headers: Optional[List[str]] = None
 in_memory_llm_clients_cache: "LLMClientCache"
diff --git a/litellm/constants.py b/litellm/constants.py
index ba5ec73d435..1dbb8a842fb 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1776,6 +1776,7 @@ DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_
 LENGTH_OF_LITELLM_GENERATED_KEY: Final = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16))
 MINIMUM_CUSTOM_KEY_LENGTH: Final = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16))
 SECRET_MANAGER_REFRESH_INTERVAL: Final = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400))
+OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: Final = frozenset({"openai", "azure"})
 LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "default_internal_user_params",
     "default_team_params",
@@ -1793,6 +1794,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     # test_general_settings_ui_fields_are_db_overridable enforces that pairing.
     "enable_anthropic_prompt_caching",
     "anthropic_prompt_caching_ttl",
+    "openai_system_messages_first",
     "max_ui_session_budget",
     "budget_rollover",
     "mcp_tool_search",
diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py
index 2485896184e..7fedefa4025 100644
--- a/litellm/litellm_core_utils/prompt_templates/common_utils.py
+++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py
@@ -2256,6 +2256,22 @@ def drop_tool_reference_parts_from_tool_messages(
     return [_drop_tool_reference_parts(message) for message in messages]  # mutable-ok: pipelines mutate message lists
 
 
+INSTRUCTION_MESSAGE_ROLES: Final = frozenset({"system", "developer"})
+
+
+def _is_instruction_message(message: AllMessageValues) -> bool:
+    return message.get("role") in INSTRUCTION_MESSAGE_ROLES
+
+
+def system_messages_first(
+    messages: list[AllMessageValues],  # mutable-ok: message pipelines type messages as mutable lists
+) -> list[AllMessageValues]:  # mutable-ok: message pipelines type messages as mutable lists
+    return [  # mutable-ok: pipelines mutate message lists
+        *(message for message in messages if _is_instruction_message(message)),
+        *(message for message in messages if not _is_instruction_message(message)),
+    ]
+
+
 def _attempt_json_repair(s: str) -> object | None:
     """
     Attempt to repair truncated JSON produced by LLM tool calls.
diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py
index ed16d7f3de0..6d17a1359bc 100644
--- a/litellm/llms/azure/chat/gpt_transformation.py
+++ b/litellm/llms/azure/chat/gpt_transformation.py
@@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     drop_tool_reference_parts_from_tool_messages,
     flatten_combinators_and_drop_non_python_regex_patterns,
     hoist_images_from_tool_messages,
+    system_messages_first,
     tool_with_sanitized_parameters,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -276,7 +277,8 @@ class AzureOpenAIConfig(BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(messages)
+        ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages
+        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages)
         azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
         return {
             "model": model,
diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py
index 9b410cf073e..9dbcf0cc089 100644
--- a/litellm/llms/openai/chat/gpt_transformation.py
+++ b/litellm/llms/openai/chat/gpt_transformation.py
@@ -12,6 +12,7 @@ from urllib.parse import urlparse
 import httpx
 
 import litellm
+from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     _extract_reasoning_content,
@@ -24,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     flatten_combinators_and_drop_non_python_regex_patterns,
     get_tool_call_names,
     hoist_images_from_tool_messages,
+    system_messages_first,
     tool_with_sanitized_parameters,
 )
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
@@ -463,6 +465,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         ]
         return MappingProxyType({"tools": sanitized})
 
+    def _prompt_cache_ordered_messages(
+        self, messages: list[AllMessageValues], litellm_params: Mapping[str, object]
+    ) -> list[AllMessageValues]:
+        if not litellm.openai_system_messages_first:
+            return messages
+        if litellm_params.get("custom_llm_provider") not in OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS:
+            return messages
+        return system_messages_first(messages)
+
     def transform_request(
         self,
         model: str,
@@ -477,7 +488,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         Returns:
             dict: The transformed request. Sent as the body of the API call.
         """
-        messages = self._transform_messages(messages=messages, model=model)
+        messages = self._transform_messages(
+            messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model
+        )
         if not self._should_preserve_cache_control_for_endpoint(
             litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
         ):
@@ -506,7 +519,9 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        transformed_messages = await self._transform_messages(messages=messages, model=model, is_async=True)
+        transformed_messages = await self._transform_messages(
+            messages=self._prompt_cache_ordered_messages(messages, litellm_params), model=model, is_async=True
+        )
         if not self._should_preserve_cache_control_for_endpoint(
             litellm_params.get("custom_llm_provider"), litellm_params.get("api_base")
         ):
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index f63e088ebf7..a375f76dbdb 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -17465,6 +17465,16 @@ _GENERAL_SETTINGS_UI_LITELLM_FIELDS: Final[dict[str, GeneralSettingsUILiteLLMFie
         "tab": "prompt_caching",
         "description": "Empty uses Anthropic's 5m default. 1h suits long sessions but doubles the cache write cost.",
     },
+    "openai_system_messages_first": {
+        "type": "Boolean",
+        "tab": "prompt_caching",
+        "description": (
+            "Moves system and developer messages to the front of the messages array on OpenAI and "
+            "Azure OpenAI chat completions requests, keeping their relative order. OpenAI's prompt cache "
+            "matches on the exact prefix, so a system message that arrives mid-conversation otherwise "
+            "breaks the cached prefix on every turn."
+        ),
+    },
     "budget_rollover": {  # mutable-ok: registry literal, frozen with its siblings below
         "type": "Boolean",
         "description": (
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
index b5890d1a5b0..c67f72680a8 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
@@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     responses_reasoning_items_from_thinking_blocks,
     split_concatenated_json_objects,
     strip_encrypted_reasoning_from_messages,
+    system_messages_first,
     update_messages_with_model_file_ids,
 )
 
@@ -1107,6 +1108,38 @@ def test_drop_tool_reference_parts_leaves_non_tool_messages_alone():
     assert result[2]["content"] == ""
 
 
+class TestSystemMessagesFirst:
+    def test_stable_partition_keeps_order_within_each_group(self):
+        messages = [
+            {"role": "user", "content": "u1"},
+            {"role": "system", "content": "s1"},
+            {"role": "assistant", "content": "a1"},
+            {"role": "developer", "content": "d1"},
+            {"role": "tool", "tool_call_id": "c1", "content": "t1"},
+            {"role": "system", "content": "s2"},
+        ]
+
+        result = system_messages_first(messages)
+
+        assert [m["content"] for m in result] == ["s1", "d1", "s2", "u1", "a1", "t1"]
+        assert [m["content"] for m in messages] == ["u1", "s1", "a1", "d1", "t1", "s2"]
+        assert all(
+            result_message is original for result_message, original in zip(result[3:], messages[::2], strict=True)
+        )
+
+    @pytest.mark.parametrize(
+        "messages",
+        [
+            [],
+            [{"role": "user", "content": "u1"}, {"role": "assistant", "content": "a1"}],
+            [{"role": "system", "content": "s1"}, {"role": "user", "content": "u1"}],
+            [{"role": "system", "content": "s1"}, {"role": "system", "content": "s2"}],
+        ],
+    )
+    def test_already_ordered_messages_come_back_unchanged(self, messages):
+        assert system_messages_first(messages) == messages
+
+
 class TestFlattenTopLevelSchemaCombinators:
     def _customer_anyof_schema(self):
         return {
diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py
index bc6cb0c0fed..e8b98c696e1 100644
--- a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py
+++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py
@@ -132,6 +132,32 @@ def test_transform_request_drops_tool_reference_parts():
     assert request["messages"][2]["content"] == ""
 
 
+@pytest.mark.parametrize(
+    "enabled, expected", [(False, ("hi", "sys", "reply", "more")), (True, ("sys", "hi", "reply", "more"))]
+)
+def test_transform_request_system_messages_first_follows_global_flag(monkeypatch, enabled, expected):
+    """Azure OpenAI shares OpenAI's prefix-matched prompt cache, so the same flag moves
+    system messages ahead of the conversation on the Azure request body."""
+    monkeypatch.setattr(litellm, "openai_system_messages_first", enabled)
+    messages = [
+        {"role": "user", "content": "hi"},
+        {"role": "system", "content": "sys"},
+        {"role": "assistant", "content": "reply"},
+        {"role": "user", "content": "more"},
+    ]
+
+    request = AzureOpenAIConfig().transform_request(
+        model="gpt-4o",
+        messages=messages,
+        optional_params={},
+        litellm_params={"custom_llm_provider": "azure"},
+        headers={},
+    )
+
+    assert tuple(m["content"] for m in request["messages"]) == expected
+    assert [m["content"] for m in messages] == ["hi", "sys", "reply", "more"]
+
+
 @pytest.mark.parametrize(
     "model, emitted_key, absent_key",
     [
diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py
index 202f81f1252..9db9ab971a0 100644
--- a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py
+++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py
@@ -68,3 +68,24 @@ def test_azure_o_series_transform_request_flattens_top_level_anyof():
     assert parameters["required"] == ["id"]
     assert "anyOf" in tool["function"]["parameters"]
     assert optional_params["tools"][0] is tool
+
+
+def test_azure_o_series_transform_request_moves_system_messages_first(monkeypatch):
+    monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+    messages = [
+        {"role": "user", "content": "hi"},
+        {"role": "developer", "content": "dev"},
+        {"role": "assistant", "content": "reply"},
+        {"role": "user", "content": "more"},
+    ]
+
+    request = AzureOpenAIO1Config().transform_request(
+        model="o3-mini",
+        messages=messages,
+        optional_params={},
+        litellm_params={"custom_llm_provider": "azure"},
+        headers={},
+    )
+
+    assert [m["content"] for m in request["messages"]] == ["dev", "hi", "reply", "more"]
+    assert [m["content"] for m in messages] == ["hi", "dev", "reply", "more"]
diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py
index b110586ae5b..53c5b9d7cbc 100644
--- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py
+++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py
@@ -1124,6 +1124,80 @@ class TestToolReferenceStripping:
         assert request["messages"][2]["content"] == ""
 
 
+class TestSystemMessagesFirst:
+    """With litellm.openai_system_messages_first on, requests bound for OpenAI put system and
+    developer messages ahead of the conversation, keeping each group's order, so the instruction
+    prefix stays byte-stable for OpenAI's prefix-matched prompt cache."""
+
+    MESSAGES: Final = (
+        {"role": "user", "content": "first turn"},
+        {"role": "system", "content": "sys 1"},
+        {"role": "assistant", "content": "reply"},
+        {"role": "developer", "content": "dev"},
+        {"role": "user", "content": "second turn"},
+        {"role": "system", "content": "sys 2"},
+    )
+    ORIGINAL_ORDER: Final = ("first turn", "sys 1", "reply", "dev", "second turn", "sys 2")
+    ORDERED: Final = ("sys 1", "dev", "sys 2", "first turn", "reply", "second turn")
+
+    def setup_method(self):
+        self.config = OpenAIGPTConfig()
+
+    def _messages(self):
+        return [dict(m) for m in self.MESSAGES]
+
+    def _transform(self, provider):
+        return self.config.transform_request(
+            model="gpt-4.1",
+            messages=self._messages(),
+            optional_params={},
+            litellm_params={"custom_llm_provider": provider},
+            headers={},
+        )
+
+    def test_default_off_keeps_caller_order(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", False)
+        assert tuple(m["content"] for m in self._transform("openai")["messages"]) == self.ORIGINAL_ORDER
+
+    def test_moves_system_and_developer_messages_first_for_openai(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        assert tuple(m["content"] for m in self._transform("openai")["messages"]) == self.ORDERED
+
+    def test_leaves_openai_compatible_providers_alone(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        assert tuple(m["content"] for m in self._transform("deepseek")["messages"]) == self.ORIGINAL_ORDER
+
+    def test_does_not_mutate_caller_messages(self, monkeypatch):
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        messages = self._messages()
+        self.config.transform_request(
+            model="gpt-4.1",
+            messages=messages,
+            optional_params={},
+            litellm_params={"custom_llm_provider": "openai"},
+            headers={},
+        )
+        assert tuple(m["content"] for m in messages) == self.ORIGINAL_ORDER
+
+    @pytest.mark.asyncio
+    async def test_async_transform_request_moves_system_messages_first(self, monkeypatch):
+        class UninstantiatedOpenAIGPTConfig(OpenAIGPTConfig):
+            _is_base_class = True
+
+            def __init__(self) -> None:
+                pass
+
+        monkeypatch.setattr(litellm, "openai_system_messages_first", True)
+        request = await UninstantiatedOpenAIGPTConfig().async_transform_request(
+            model="gpt-4.1",
+            messages=self._messages(),
+            optional_params={},
+            litellm_params={"custom_llm_provider": "openai"},
+            headers={},
+        )
+        assert tuple(m["content"] for m in request["messages"]) == self.ORDERED
+
+
 class TestOpenAIPromptCacheBreakpointChatPath:
     """Chat-path shape for OpenAI explicit prompt caching (#37509)."""
 
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 42af8e0af21..c5f08632c43 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -10750,6 +10750,7 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
     monkeypatch.setattr(ps, "prisma_client", mock_prisma)
     monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
     monkeypatch.setattr(litellm, "anthropic_prompt_caching_ttl", "1h")
+    monkeypatch.setattr(litellm, "openai_system_messages_first", False)
     app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
         user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN
     )
@@ -10771,6 +10772,10 @@ def test_get_config_list_includes_anthropic_prompt_caching_fields(monkeypatch):
         assert fields["enable_anthropic_prompt_caching"]["field_tab"] == "prompt_caching"
         assert fields["anthropic_prompt_caching_ttl"]["field_tab"] == "prompt_caching"
         assert fields["budget_exceeded_throttle_percentage"]["field_tab"] is None
+
+        assert fields["openai_system_messages_first"]["field_type"] == "Boolean"
+        assert fields["openai_system_messages_first"]["field_value"] is False
+        assert fields["openai_system_messages_first"]["field_tab"] == "prompt_caching"
     finally:
         app.dependency_overrides.clear()
 
@@ -10887,6 +10892,7 @@ def test_general_settings_ui_defaults_unchanged_for_existing_fields():
     [
         ("enable_anthropic_prompt_caching", True),
         ("anthropic_prompt_caching_ttl", "1h"),
+        ("openai_system_messages_first", True),
     ],
 )
 def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_name, db_value):
@@ -10945,6 +10951,8 @@ def test_get_config_list_marks_untouched_prompt_caching_flag_as_not_set(monkeypa
         ("enable_anthropic_prompt_caching", False),
         ("anthropic_prompt_caching_ttl", "5m"),
         ("anthropic_prompt_caching_ttl", "1h"),
+        ("openai_system_messages_first", True),
+        ("openai_system_messages_first", False),
     ],
 )
 @pytest.mark.asyncio
@@ -10993,6 +11001,8 @@ async def test_update_config_field_prompt_caching_persists_to_litellm_settings(m
         ("anthropic_prompt_caching_ttl", "10m"),
         ("anthropic_prompt_caching_ttl", "1H"),
         ("anthropic_prompt_caching_ttl", 3600),
+        ("openai_system_messages_first", "yes"),
+        ("openai_system_messages_first", 1),
     ],
 )
 @pytest.mark.asyncio
@@ -11032,6 +11042,7 @@ async def test_update_config_field_prompt_caching_rejects_invalid(monkeypatch, f
     [
         ("enable_anthropic_prompt_caching", False),
         ("anthropic_prompt_caching_ttl", None),
+        ("openai_system_messages_first", False),
         ("budget_exceeded_throttle_percentage", None),
     ],
 )
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx
index 9cd1444b0b9..b4df567e250 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.integration.test.tsx
@@ -45,6 +45,15 @@ const SETTINGS_FIXTURE = [
     field_tab: "prompt_caching",
     field_default_value: null,
   },
+  {
+    field_name: "openai_system_messages_first",
+    field_type: "Boolean",
+    field_value: false,
+    field_description: "openai system first toggle",
+    stored_in_db: null,
+    field_tab: "prompt_caching",
+    field_default_value: false,
+  },
   {
     field_name: "max_ui_session_budget",
     field_type: "Dollar",
@@ -101,6 +110,39 @@ describe("GeneralSettings General tab", () => {
   });
 });
 
+describe("GeneralSettings Prompt Caching tab", () => {
+  beforeEach(() => {
+    vi.mocked(getGeneralSettingsCall).mockResolvedValue([...SETTINGS_FIXTURE.map((s) => ({ ...s }))]);
+    vi.mocked(updateConfigFieldSetting).mockClear();
+    vi.mocked(deleteConfigFieldSetting).mockClear();
+  });
+
+  it("persists openai_system_messages_first when its switch is turned on", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(<GeneralSettings accessToken="token" userRole="Admin" userID="user" />);
+
+    await user.click(await screen.findByRole("tab", { name: "Prompt Caching" }));
+    const toggle = await screen.findByRole("switch", { name: "System messages first for OpenAI" });
+    expect(toggle).not.toBeChecked();
+
+    await user.click(toggle);
+
+    expect(toggle).toBeChecked();
+    expect(updateConfigFieldSetting).toHaveBeenCalledWith("token", "openai_system_messages_first", true);
+    expect(deleteConfigFieldSetting).not.toHaveBeenCalled();
+  });
+
+  it("keeps the prompt caching rows off the General tab table", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(<GeneralSettings accessToken="token" userRole="Admin" userID="user" />);
+
+    await user.click(screen.getByText("General"));
+    await settingsRow("max_ui_session_budget");
+
+    expect(screen.queryByText("openai_system_messages_first")).not.toBeInTheDocument();
+  });
+});
+
 // The five tabs here are proxy-wide settings. Auto-routers moved to Models + Endpoints.
 describe("GeneralSettings tabs", () => {
   beforeEach(() => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx
index df9e328ec3b..9a718cbe9b8 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx
@@ -18,6 +18,9 @@ import RoutingGroups from "@/components/routing_groups";
 const PROMPT_CACHING_TAB = "prompt_caching";
 const ENABLE_ANTHROPIC_PROMPT_CACHING = "enable_anthropic_prompt_caching";
 const ANTHROPIC_PROMPT_CACHING_TTL = "anthropic_prompt_caching_ttl";
+const OPENAI_SYSTEM_MESSAGES_FIRST = "openai_system_messages_first";
+
+const isOn = (value: unknown) => value === true || value === "true";
 
 interface GeneralSettingsPageProps {
   accessToken: string | null;
@@ -117,14 +120,15 @@ export const PromptCachingPanel: React.FC<{
 }> = ({ accessToken, settings, onChange }) => {
   const enableSetting = settings.find((s) => s.field_name === ENABLE_ANTHROPIC_PROMPT_CACHING);
   const ttlSetting = settings.find((s) => s.field_name === ANTHROPIC_PROMPT_CACHING_TTL);
+  const systemFirstSetting = settings.find((s) => s.field_name === OPENAI_SYSTEM_MESSAGES_FIRST);
 
-  // The two rows come from the same registry the General tab reads; if they
+  // The rows come from the same registry the General tab reads; if they
   // are not loaded yet there is nothing to render.
   if (!enableSetting) {
     return null;
   }
 
-  const enabled = enableSetting.field_value === true || enableSetting.field_value === "true";
+  const enabled = isOn(enableSetting.field_value);
 
   // Apply immediately: a toggle and a dropdown are direct controls, so there is
   // no separate Update button. Clearing the ttl resets it to the provider default.
@@ -175,6 +179,20 @@ export const PromptCachingPanel: React.FC<{
             </Select>
           </div>
         )}
+
+        {systemFirstSetting && (
+          <div className="mt-6 flex items-start justify-between gap-8">
+            <div className="min-w-0 max-w-2xl">
+              <p className="font-medium">System messages first for OpenAI</p>
+              <p className="mt-1 break-words text-xs text-muted-foreground">{systemFirstSetting.field_description}</p>
+            </div>
+            <Switch
+              aria-label="System messages first for OpenAI"
+              checked={isOn(systemFirstSetting.field_value)}
+              onCheckedChange={(checked) => persist(OPENAI_SYSTEM_MESSAGES_FIRST, checked)}
+            />
+          </div>
+        )}
       </CardContent>
     </Card>
   );

From 99fa38504a5d90e792667909bb596fd4c9003272 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:55:07 +0000
Subject: [PATCH 28/67] fix(ui): bound Models table page, page size and sort_by
 read from the URL

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../components/AllModelsTab.test.tsx          | 19 +++++++++--
 .../components/AllModelsTab.tsx               | 34 +++++++++++++------
 .../components/ModelsTableColumns.tsx         | 13 +++++++
 3 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx
index 7e45ff6834b..a5eb149e1f0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx
@@ -317,13 +317,28 @@ describe("AllModelsTab", () => {
       expect(screen.queryByText(/To access these models/)).not.toBeInTheDocument();
     });
 
-    it("falls back to the first page and default size when the URL carries values the server rejects", () => {
-      renderWithProviders(<AllModelsTab {...defaultProps} />, { searchParams: { page: "0", page_size: "-5" } });
+    it("clamps a hand-edited page and page size into the range the table supports", () => {
+      renderWithProviders(<AllModelsTab {...defaultProps} />, { searchParams: { page: "0", page_size: "5000" } });
 
       expect(lastModelsInfoCall().page).toBe(1);
+      expect(lastModelsInfoCall().size).toBe(100);
+    });
+
+    it("keeps the default page size when the URL value is not a number", () => {
+      renderWithProviders(<AllModelsTab {...defaultProps} />, { searchParams: { page_size: "lots" } });
+
       expect(lastModelsInfoCall().size).toBe(50);
     });
 
+    it("ignores a sort_by the table cannot sort by instead of forwarding it to the server", () => {
+      renderWithProviders(<AllModelsTab {...defaultProps} />, {
+        searchParams: { sort_by: "litellm_credential_name", sort_order: "desc" },
+      });
+
+      expect(lastModelsInfoCall().sortBy).toBeUndefined();
+      expect(lastModelsInfoCall().sortOrder).toBeUndefined();
+    });
+
     it("writes sort changes to the URL with the page cleared", async () => {
       setModelsInfo([makeRow()], 200);
       const user = userEvent.setup();
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
index efe74a273d6..2217bca0fa0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx
@@ -13,7 +13,7 @@ import { useQueryClient } from "@tanstack/react-query";
 import { useDebouncedValue } from "@tanstack/react-pacer/debouncer";
 import { ColumnFiltersState, functionalUpdate, OnChangeFn, PaginationState, SortingState } from "@tanstack/react-table";
 import { Info } from "lucide-react";
-import { parseAsInteger, parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs";
+import { createParser, parseAsInteger, parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs";
 import { useCallback, useMemo, useState } from "react";
 
 import { useModelsInfo } from "../../hooks/models/useModels";
@@ -25,22 +25,39 @@ import {
   PERSONAL_TEAM_VALUE,
   WILDCARD_MODEL_GROUP_VALUE,
 } from "./AllModelsTable";
-import { ACCESS_GROUPS_COLUMN_ID, MODEL_NAME_COLUMN_ID, toServerSortField } from "./ModelsTableColumns";
+import {
+  ACCESS_GROUPS_COLUMN_ID,
+  isModelTableSortColumnId,
+  MODEL_NAME_COLUMN_ID,
+  MODEL_TABLE_SORT_COLUMN_IDS,
+  toServerSortField,
+} from "./ModelsTableColumns";
 
 const SEARCH_DEBOUNCE_WAIT_MS = 200;
 const DEFAULT_PAGE_SIZE = 50;
+const MAX_PAGE_SIZE = 100;
+const MAX_PAGE = 100_000;
 
 const MODEL_VIEW_MODES = ["current_team", "all"] as const satisfies readonly ModelViewMode[];
 
+const boundedInteger = (min: number, max: number, fallback: number) =>
+  createParser({
+    parse: (value: string) => {
+      const parsed = parseAsInteger.parse(value);
+      return parsed === null ? null : Math.min(Math.max(parsed, min), max);
+    },
+    serialize: String,
+  }).withDefault(fallback);
+
 const TABLE_STATE = {
   model_search: parseAsString.withDefault(""),
   view_mode: parseAsStringLiteral(MODEL_VIEW_MODES).withDefault("current_team"),
   filter_team: parseAsString.withDefault(PERSONAL_TEAM_VALUE),
   access_group: parseAsString.withDefault(""),
-  sort_by: parseAsString.withDefault(""),
+  sort_by: parseAsStringLiteral(MODEL_TABLE_SORT_COLUMN_IDS),
   sort_order: parseAsStringLiteral(["asc", "desc"] as const).withDefault("asc"),
-  page: parseAsInteger.withDefault(1),
-  page_size: parseAsInteger.withDefault(DEFAULT_PAGE_SIZE),
+  page: boundedInteger(1, MAX_PAGE, 1),
+  page_size: boundedInteger(1, MAX_PAGE_SIZE, DEFAULT_PAGE_SIZE),
 };
 
 interface AllModelsTabProps {
@@ -72,10 +89,7 @@ const AllModelsTab = ({
   const selectedTeamValue = tableState.filter_team;
   const selectedModelAccessGroupFilter = tableState.access_group || null;
   const pagination = useMemo<PaginationState>(
-    () => ({
-      pageIndex: Math.max(tableState.page, 1) - 1,
-      pageSize: tableState.page_size >= 1 ? tableState.page_size : DEFAULT_PAGE_SIZE,
-    }),
+    () => ({ pageIndex: tableState.page - 1, pageSize: tableState.page_size }),
     [tableState.page, tableState.page_size],
   );
   const sorting = useMemo<SortingState>(
@@ -177,7 +191,7 @@ const AllModelsTab = ({
   const handleSortingChange: OnChangeFn<SortingState> = (updater) => {
     const active = functionalUpdate(updater, sorting)[0];
     void setTableState({
-      sort_by: active?.id ?? null,
+      sort_by: active && isModelTableSortColumnId(active.id) ? active.id : null,
       sort_order: active?.desc ? "desc" : null,
       page: null,
     });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx
index 0cc1207e547..c5bab598a8b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelsTableColumns.tsx
@@ -24,6 +24,19 @@ export const TEAM_ID_COLUMN_ID = "model_info_team_id";
 export const ACCESS_GROUPS_COLUMN_ID = "model_info_access_groups";
 export const STATUS_COLUMN_ID = "model_info_db_model";
 
+export const MODEL_TABLE_SORT_COLUMN_IDS = [
+  MODEL_NAME_COLUMN_ID,
+  CREATED_BY_COLUMN_ID,
+  UPDATED_AT_COLUMN_ID,
+  COSTS_COLUMN_ID,
+  STATUS_COLUMN_ID,
+] as const;
+
+export type ModelTableSortColumnId = (typeof MODEL_TABLE_SORT_COLUMN_IDS)[number];
+
+export const isModelTableSortColumnId = (columnId: string): columnId is ModelTableSortColumnId =>
+  (MODEL_TABLE_SORT_COLUMN_IDS as readonly string[]).includes(columnId);
+
 const COLUMN_ID_TO_SERVER_SORT_FIELD: Record<string, string> = {
   [COSTS_COLUMN_ID]: "costs",
   [STATUS_COLUMN_ID]: "status",

From 3cb64978d47ea6d7b5cfe28bb296cf20a7a5eebb Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:55:37 +0000
Subject: [PATCH 29/67] fix(proxy): honor LITELLM_LOG for uvicorn and proxy
 extras loggers

LITELLM_LOG=ERROR still printed INFO lines from uvicorn (startup and access log) and from the litellm_proxy_extras migration logger, because neither read the variable. Forward the resolved level to uvicorn when LITELLM_LOG is set and no explicit log_config or JSON logging is in use, and let the extras logger take its level from LITELLM_LOG

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../litellm_proxy_extras/_logging.py          |  2 +-
 litellm-proxy-extras/tests/test_logging.py    | 34 +++++++++++++++++++
 litellm/_logging.py                           |  2 +-
 litellm/proxy/proxy_cli.py                    |  4 ++-
 tests/test_litellm/proxy/test_proxy_cli.py    | 33 ++++++++++++++++++
 5 files changed, 72 insertions(+), 3 deletions(-)
 create mode 100644 litellm-proxy-extras/tests/test_logging.py

diff --git a/litellm-proxy-extras/litellm_proxy_extras/_logging.py b/litellm-proxy-extras/litellm_proxy_extras/_logging.py
index ecf467fbf45..64e07a180d3 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/_logging.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/_logging.py
@@ -40,4 +40,4 @@ if not logger.handlers:
             logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
         )
     logger.addHandler(handler)
-    logger.setLevel(logging.INFO)
+    logger.setLevel(os.getenv("LITELLM_LOG", "INFO").upper())
diff --git a/litellm-proxy-extras/tests/test_logging.py b/litellm-proxy-extras/tests/test_logging.py
new file mode 100644
index 00000000000..b54d4fb72c9
--- /dev/null
+++ b/litellm-proxy-extras/tests/test_logging.py
@@ -0,0 +1,34 @@
+import importlib
+import logging
+
+import pytest
+
+import litellm_proxy_extras._logging as extras_logging
+
+
+def test_litellm_log_error_silences_extras_info_lines(monkeypatch):
+    saved_handlers = logging.getLogger("litellm_proxy_extras").handlers[:]
+    monkeypatch.setenv("LITELLM_LOG", "ERROR")
+    logging.getLogger("litellm_proxy_extras").handlers[:] = []
+    try:
+        reloaded = importlib.reload(extras_logging).logger
+        assert reloaded.isEnabledFor(logging.INFO) is False
+        assert reloaded.isEnabledFor(logging.ERROR) is True
+    finally:
+        logging.getLogger("litellm_proxy_extras").handlers[:] = saved_handlers
+        logging.getLogger("litellm_proxy_extras").setLevel(logging.INFO)
+
+
+@pytest.mark.parametrize("litellm_log", [None, "info", "DEBUG"])
+def test_unset_or_verbose_litellm_log_keeps_extras_info_lines(monkeypatch, litellm_log):
+    saved_handlers = logging.getLogger("litellm_proxy_extras").handlers[:]
+    if litellm_log is None:
+        monkeypatch.delenv("LITELLM_LOG", raising=False)
+    else:
+        monkeypatch.setenv("LITELLM_LOG", litellm_log)
+    logging.getLogger("litellm_proxy_extras").handlers[:] = []
+    try:
+        assert importlib.reload(extras_logging).logger.isEnabledFor(logging.INFO) is True
+    finally:
+        logging.getLogger("litellm_proxy_extras").handlers[:] = saved_handlers
+        logging.getLogger("litellm_proxy_extras").setLevel(logging.INFO)
diff --git a/litellm/_logging.py b/litellm/_logging.py
index 03a9bcf21cf..893133d62fe 100644
--- a/litellm/_logging.py
+++ b/litellm/_logging.py
@@ -404,7 +404,7 @@ def _parse_json_logs_env(value: str | None) -> bool:
 json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
-numeric_level: Final[str] = getattr(logging, log_level.upper())
+numeric_level: Final[int] = getattr(logging, log_level.upper())
 handler: Final = LevelRoutingStreamHandler()
 handler.setLevel(numeric_level)
 handler.addFilter(_secret_filter)
diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py
index 01a3da08998..6a23b422717 100644
--- a/litellm/proxy/proxy_cli.py
+++ b/litellm/proxy/proxy_cli.py
@@ -261,7 +261,7 @@ class ProxyInitializationHelpers:
         import uvicorn
 
         import litellm
-        from litellm._logging import _get_uvicorn_json_log_config
+        from litellm._logging import _get_uvicorn_json_log_config, numeric_level
 
         uvicorn_args: Final = {
             "app": "litellm.proxy.proxy_server:app",
@@ -275,6 +275,8 @@ class ProxyInitializationHelpers:
         elif litellm.json_logs:
             # Use JSON log config for uvicorn to ensure all logs (including exceptions) are JSON
             uvicorn_args["log_config"] = _get_uvicorn_json_log_config()
+        elif os.environ.get("LITELLM_LOG"):
+            uvicorn_args["log_level"] = numeric_level
         if keepalive_timeout is not None:
             uvicorn_args["timeout_keep_alive"] = keepalive_timeout
         if timeout_worker_healthcheck is not None:
diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py
index e25e6a59884..8f3c21e10c2 100644
--- a/tests/test_litellm/proxy/test_proxy_cli.py
+++ b/tests/test_litellm/proxy/test_proxy_cli.py
@@ -139,6 +139,39 @@ class TestProxyInitializationHelpers:
             )
             assert args["timeout_worker_healthcheck"] == 15
 
+    @staticmethod
+    def _uvicorn_access_info_enabled(args: dict) -> bool:
+        import logging
+
+        names = ("uvicorn", "uvicorn.error", "uvicorn.access", "uvicorn.asgi")
+        saved = tuple((logging.getLogger(n), logging.getLogger(n).handlers[:], logging.getLogger(n).level) for n in names)
+        try:
+            uvicorn.Config(**args).configure_logging()
+            return logging.getLogger("uvicorn.access").isEnabledFor(logging.INFO)
+        finally:
+            for lg, handlers, level in saved:
+                lg.handlers[:] = handlers
+                lg.setLevel(level)
+
+    def test_litellm_log_error_silences_uvicorn_info_lines(self, monkeypatch):
+        import logging
+
+        monkeypatch.setenv("LITELLM_LOG", "ERROR")
+        with patch(  # test-quality-ok: numeric_level is resolved from LITELLM_LOG once at import; no other way to set it
+            "litellm._logging.numeric_level", logging.ERROR
+        ):
+            args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
+
+        assert "log_config" not in args
+        assert self._uvicorn_access_info_enabled(args) is False
+
+    def test_unset_litellm_log_keeps_uvicorn_default_info_lines(self, monkeypatch):
+        monkeypatch.delenv("LITELLM_LOG", raising=False)
+        args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
+
+        assert "log_level" not in args
+        assert self._uvicorn_access_info_enabled(args) is True
+
     def test_installed_uvicorn_supports_worker_flags(self):
         params = inspect.signature(uvicorn.Config.__init__).parameters
         assert "timeout_worker_healthcheck" in params

From 5237fe4df3c8fc290e4f1f66cdeb18d047e26776 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:57:12 +0000
Subject: [PATCH 30/67] refactor(passthrough): read the deployment model_info
 request state in two steps

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/pass_through_endpoints/pass_through_endpoints.py | 3 ++-
 litellm/types/passthrough_endpoints/pass_through_endpoints.py  | 3 +--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index 5d5275abad0..686544d352c 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -614,8 +614,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         _metadata.update(
             LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
         )
+        _request_state: Final = getattr(request, "state", None)
         deployment_model_info: Final = getattr(
-            getattr(request, "state", None), LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, None
+            _request_state, LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, None
         )
         if isinstance(deployment_model_info, Mapping):
             _metadata["model_info"] = dict(deployment_model_info)
diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py
index fb12daab199..e47acf9d68b 100644
--- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py
+++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py
@@ -11,8 +11,7 @@ LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY: Final = "litellm_pass_through_custom
 # exact byte/string body, such as AWS SigV4-signed requests.
 LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY: Final = "litellm_pass_through_raw_body"
 
-# Request.state key carrying the `model_info` of the router deployment a provider
-# route resolved (e.g. Vertex), so logging attributes the call to that deployment.
+# `model_info` of the router deployment a provider route (e.g. Vertex) resolved for this request.
 LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY: Final = "litellm_pass_through_deployment_model_info"
 
 # Attribute set on the FastAPI endpoint function of every user-defined pass-through

From c49fb1dd9d17e4a3b7c99b62662f9e33dad43c42 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:58:35 +0000
Subject: [PATCH 31/67] fix(openai): drop env var read for
 openai_system_messages_first, config and Admin UI set it

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index 55e258a2c27..3668e6efb0c 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -343,7 +343,7 @@ _anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_
 anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
     "1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
 )
-openai_system_messages_first: bool = os.getenv("LITELLM_OPENAI_SYSTEM_MESSAGES_FIRST", "false").lower() == "true"
+openai_system_messages_first: bool = False
 disable_vertex_batch_output_transformation: bool = False
 extra_spend_tag_headers: Optional[List[str]] = None
 in_memory_llm_clients_cache: "LLMClientCache"

From bae2bf003e26048bea1500efef65a95bbd57dd1e Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:52:20 +0000
Subject: [PATCH 32/67] fix(proxy): resolve router_settings.model_group_alias
 before key/team model auth

Key and team router_settings.model_group_alias aliases were resolved only after the key/team model allowlist checks ran, so a key allowed the alias target was denied when it requested the alias. Resolve the alias during auth and rewrite the request body to the target before the allowlist checks. The alias the client sent is kept in the request scope so the response model still echoes it.

Resolves LIT-3054

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/constants.py                          |  1 +
 litellm/proxy/auth/user_api_key_auth.py       | 49 +++++++++-
 litellm/proxy/common_request_processing.py    | 12 +--
 .../proxy/common_utils/http_parsing_utils.py  |  9 +-
 .../proxy/auth/test_user_api_key_auth.py      | 97 +++++++++++++++++++
 .../proxy/test_common_request_processing.py   | 53 +++++++++-
 6 files changed, 212 insertions(+), 9 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index ba5ec73d435..ca2be8af5fe 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -317,6 +317,7 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
 BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
 BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
+CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model"
 REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
 REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
 
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 7d62baf39a8..a02661db9ec 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -26,6 +26,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
 from litellm.caching.redis_cache import RedisCache
 from litellm.constants import (
+    CLIENT_REQUESTED_MODEL_SCOPE_KEY,
     GLOBAL_PROXY_SPEND_CACHE_KEY,
     INVALID_VIRTUAL_KEY_ERROR_MARKER,
     INVALID_VIRTUAL_KEY_ERROR_MESSAGE,
@@ -124,6 +125,7 @@ from litellm.proxy.utils import (
     normalize_route_for_root_path,
 )
 from litellm.repositories.table_repositories import TeamMembershipRepository
+from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.services import ServiceTypes
 
@@ -235,13 +237,56 @@ async def _normalize_claude_model(
         request.scope[_CLAUDE_MODEL_NORMALIZED] = True
     if source is None:
         return
-    request_data["model"] = source
+    _rewrite_request_model(request_data, request, source)
+
+
+def _rewrite_request_model(
+    request_data: dict,  # mutable-ok: the request body is rewritten in place for every downstream reader
+    request: Request | None,
+    model: str,
+) -> None:
+    request_data["model"] = model
     _safe_set_request_parsed_body(request=request, parsed_body=request_data)
     if request is not None:
         request._json = request_data
         request._body = orjson.dumps(request_data)
 
 
+_MODEL_GROUP_ALIAS_RESOLVED: Final = "litellm.model_group_alias_resolved"
+
+
+async def _resolve_router_settings_model_group_alias(
+    request_data: dict,  # mutable-ok: the request body is rewritten in place for every downstream reader
+    valid_token: UserAPIKeyAuth,
+    request: Request | None,
+    route: str,
+) -> None:
+    """Rewrite the requested model through the key's or team's ``router_settings.model_group_alias``
+    before the allowlist checks, so they authorize the model group the request is routed to.
+    """
+    from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj
+
+    if request is None or llm_router is None or not RouteChecks.is_llm_api_route(route=route):
+        return
+    if request.scope.get(_MODEL_GROUP_ALIAS_RESOLVED) is True:
+        return
+    request.scope[_MODEL_GROUP_ALIAS_RESOLVED] = True
+    requested: Final = request_data.get("model")
+    if not isinstance(requested, str) or await read_raw_json_body(request=request) is None:
+        return
+    settings: Final = await proxy_config.get_hierarchical_router_settings(
+        user_api_key_dict=valid_token, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
+    )
+    if not isinstance(settings, Mapping):
+        return
+    target: Final = resolve_model_group_alias(settings.get("model_group_alias"), requested)
+    if target is None or target == requested:
+        return
+    verbose_proxy_logger.debug("router_settings.model_group_alias resolved %s -> %s before auth", requested, target)
+    request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested)
+    _rewrite_request_model(request_data, request, target)
+
+
 def _get_model_names_for_budget_checks(
     model: str | list[str] | None,
 ) -> list[str]:
@@ -2926,6 +2971,7 @@ async def _authorize_authenticated_request(
     ## ENSURE DISABLE ROUTE WORKS ACROSS ALL USER AUTH FLOWS ##
     RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request)
     await _normalize_claude_model(request_data, user_api_key_auth_obj, request, route)
+    await _resolve_router_settings_model_group_alias(request_data, user_api_key_auth_obj, request, route)
 
     # Single authorization point. Builder paths MUST NOT call common_checks.
     # Route through the same exception handler the builder uses so
@@ -3312,6 +3358,7 @@ async def _enforce_key_and_fallback_model_access(
     Not included in common_checks — common_checks enforces team/user/project model access only.
     """
     await _normalize_claude_model(request_data, valid_token, request, route)
+    await _resolve_router_settings_model_group_alias(request_data, valid_token, request, route)
     config: Final = valid_token.config
 
     if config != {}:
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 4a4daa68cce..3e3183d1293 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -56,6 +56,7 @@ from litellm.proxy.common_utils.callback_utils import (
     get_logging_caching_headers,
     get_remaining_tokens_and_requests_from_request_data,
 )
+from litellm.proxy.common_utils.http_parsing_utils import get_client_requested_model
 from litellm.proxy.common_utils.openai_error_payload import (
     attribute_of,
     error_status_code,
@@ -622,9 +623,9 @@ async def _resolve_per_request_model_group_alias(
     holds the global config map and is shared across requests, so a per-request
     map has to be applied here instead of being forwarded to the Router.
 
-    Model access was authorized against the requested group, so the target is
-    authorized in its own right before the rewrite; a key that may not call the
-    target gets the usual 403 rather than being quietly served it.
+    Auth already rewrote the body through this map for LLM API routes, so this is
+    a fallback for callers that skipped it; the target is authorized in its own
+    right before the rewrite, so a key that may not call it gets the usual 403.
 
     Returns the target model group, or None when no alias applies.
     """
@@ -2338,9 +2339,8 @@ class ProxyBaseLLMRequestProcessing:
         """
         Common request processing logic for both chat completions and responses API endpoints
         """
-        requested_model_from_client: Final[str | None] = (
-            self.data.get("model") if isinstance(self.data.get("model"), str) else None
-        )
+        client_model: Final = get_client_requested_model(request) or self.data.get("model")
+        requested_model_from_client: Final[str | None] = client_model if isinstance(client_model, str) else None
         self._debug_log_request_payload()
 
         if skip_pre_call_logic:
diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py
index 9c2767c7771..ec2e05541cb 100644
--- a/litellm/proxy/common_utils/http_parsing_utils.py
+++ b/litellm/proxy/common_utils/http_parsing_utils.py
@@ -9,7 +9,7 @@ from fastapi import Request, UploadFile, status
 from typing_extensions import NotRequired, ReadOnly, Required
 
 from litellm._logging import verbose_proxy_logger
-from litellm.constants import MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
+from litellm.constants import CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB
 from litellm.proxy._types import ProxyException
 from litellm.proxy.common_utils.callback_utils import (
     get_metadata_variable_name_from_kwargs,
@@ -235,6 +235,13 @@ def _safe_get_request_parsed_body(request: Request | None) -> dict | None:
     return None
 
 
+def get_client_requested_model(request: Request | None) -> str | None:
+    if request is None or not hasattr(request, "scope"):
+        return None
+    model: Final = request.scope.get(CLIENT_REQUESTED_MODEL_SCOPE_KEY)
+    return model if isinstance(model, str) else None
+
+
 def _safe_get_request_query_params(request: Request | None) -> dict:
     if request is None:
         return {}
diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
index 866ea0b20e4..ded45f756be 100644
--- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
+++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
@@ -34,6 +34,7 @@ from litellm.proxy._types import (
 from litellm.proxy.auth.handle_jwt import JWTHandler
 from litellm.proxy.auth.auth_checks import TeamNotFoundError, UserNotFoundError, get_key_object, _cache_key_object
 from litellm.proxy.auth.route_checks import RouteChecks
+from litellm.proxy.common_utils.http_parsing_utils import get_client_requested_model
 from litellm.proxy.auth.user_api_key_auth import (
     _check_key_model_budget_with_fallback,
     _ensure_litellm_received_at_on_request_state,
@@ -8137,3 +8138,99 @@ async def test_auth_flow_enters_virtual_key_mapping_when_only_an_issuer_configur
     assert resolve_mock.await_args.kwargs["jwt_claims"][JWTHandler.LITELLM_JWT_ISSUER_CLAIM] == ISSUER_TWO
     assert result.api_key == "hashed-mapped-key"
     assert result.team_id == "svc-team"
+
+
+def _alias_router() -> litellm.Router:
+    return litellm.Router(model_list=[{"model_name": name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}} for name in ("claude-haiku", "claude-sonnet")])
+
+
+def _alias_request(route: str, data: dict, content_type: str = "application/json"):
+    """A request as auth sees it: the body already read once and cached alongside its parsed form."""
+    from starlette.requests import Request
+
+    headers = [(b"content-type", content_type.encode())]
+    request = Request({"type": "http", "method": "POST", "path": route, "headers": headers, "query_string": b"", "parsed_body": (tuple(data), data)})
+    request._body = json.dumps(data).encode()
+    return request
+
+
+def _alias_token(monkeypatch, level: str, alias: dict, models: list) -> UserAPIKeyAuth:
+    """A key whose ``router_settings.model_group_alias`` lives on the key itself or on its cached team row."""
+    from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+
+    if level == "key":
+        return UserAPIKeyAuth(models=models, router_settings={"model_group_alias": alias})
+    cache = UserApiKeyCache()
+    cache.set_cache(key="team_id:team-alias", value=LiteLLM_TeamTableCachedObj(team_id="team-alias", models=models, router_settings={"model_group_alias": alias}))
+    monkeypatch.setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
+    monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", MagicMock())
+    return UserAPIKeyAuth(team_id="team-alias", models=models)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("level", ["key", "team"])
+@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/messages", "/v1/embeddings"])
+async def test_router_settings_model_group_alias_authorizes_target_for_key(monkeypatch, level, route):
+    """LIT-3054: a key allowed only the alias target must be able to call the alias, and a key not
+    allowed the target must still be denied even when the alias itself is what it requested."""
+    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
+
+    router = _alias_router()
+    monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
+    data = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]}
+    request = _alias_request(route, data)
+    token = _alias_token(monkeypatch, level, {"AgentX-LLM": "claude-haiku"}, ["claude-haiku"])
+    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route=route, request=request, llm_model_list=router.model_list, llm_router=router)
+    assert data["model"] == "claude-haiku"
+    assert (await request.json())["model"] == "claude-haiku"
+    assert json.loads(await request.body())["model"] == "claude-haiku"
+    assert request.scope["parsed_body"][1]["model"] == "claude-haiku"
+    assert get_client_requested_model(request) == "AgentX-LLM"
+
+    denied = _alias_token(monkeypatch, level, {"AgentX-LLM": "claude-sonnet"}, ["claude-haiku"])
+    denied_data = {"model": "AgentX-LLM"}
+    with pytest.raises(ProxyException) as exc:
+        await _enforce_key_and_fallback_model_access(valid_token=denied, request_data=denied_data, route=route, request=_alias_request(route, denied_data), llm_model_list=router.model_list, llm_router=router)
+    assert "claude-sonnet" in exc.value.message
+
+
+@pytest.mark.asyncio
+async def test_router_settings_model_group_alias_leaves_form_bodies_alone(monkeypatch):
+    """LIT-3054: a multipart body cannot be re-serialized as JSON, so auth must not rewrite it."""
+    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
+
+    router = _alias_router()
+    monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
+    data = {"model": "AgentX-LLM"}
+    request = _alias_request("/v1/audio/transcriptions", data, content_type="multipart/form-data; boundary=x")
+    token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku", "AgentX-LLM"])
+    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route="/v1/audio/transcriptions", request=request, llm_model_list=router.model_list, llm_router=router)
+    assert data["model"] == "AgentX-LLM"
+    assert get_client_requested_model(request) is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("target, expect_denied", [("claude-haiku", False), ("claude-sonnet", True)])
+async def test_router_settings_model_group_alias_authorizes_target_for_team(monkeypatch, target, expect_denied):
+    """LIT-3054: the team allowlist check in common_checks must judge the alias target, not the alias."""
+    import litellm.proxy.proxy_server as _proxy_server_mod
+    from litellm.proxy.auth.user_api_key_auth import _authorize_authenticated_request
+
+    router = _alias_router()
+    logging_obj = MagicMock(post_call_failure_hook=AsyncMock(return_value=None))
+    attrs = {**_proxy_attrs_for_centralized_checks(), "llm_router": router, "proxy_logging_obj": logging_obj}
+    for k, v in attrs.items():
+        monkeypatch.setattr(_proxy_server_mod, k, v)
+    token = _alias_token(monkeypatch, "team", {"AgentX-LLM": target}, ["claude-haiku"])
+    token.team_models = ["claude-haiku"]
+    data = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]}
+    request = _alias_request("/v1/chat/completions", data)
+    if expect_denied:
+        with pytest.raises(ProxyException) as exc:
+            await _authorize_authenticated_request(user_api_key_auth_obj=token, request=request, request_data=data, route="/v1/chat/completions", api_key="sk-test")
+        assert exc.value.type == ProxyErrorTypes.team_model_access_denied
+        assert target in exc.value.message
+        return
+    await _authorize_authenticated_request(user_api_key_auth_obj=token, request=request, request_data=data, route="/v1/chat/completions", api_key="sk-test")
+    assert (await request.json())["model"] == target
+    assert get_client_requested_model(request) == "AgentX-LLM"
diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py
index cabfcc9918f..e7b83455a9c 100644
--- a/tests/test_litellm/proxy/test_common_request_processing.py
+++ b/tests/test_litellm/proxy/test_common_request_processing.py
@@ -13,7 +13,11 @@ from fastapi.responses import JSONResponse, StreamingResponse
 
 import litellm
 from litellm._uuid import uuid
-from litellm.constants import MAX_LITELLM_CALL_ID_LENGTH, RETURN_RAW_MODEL_NAME_METADATA_KEY
+from litellm.constants import (
+    CLIENT_REQUESTED_MODEL_SCOPE_KEY,
+    MAX_LITELLM_CALL_ID_LENGTH,
+    RETURN_RAW_MODEL_NAME_METADATA_KEY,
+)
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.opentelemetry import UserAPIKeyAuth
 from litellm.proxy.common_request_processing import (
@@ -4395,6 +4399,53 @@ class TestDisconnectGatherCleanup:
             )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("client_model, expected", [("AgentX-LLM", "AgentX-LLM"), (None, "gpt-mini")])
+async def test_response_model_echoes_the_name_the_client_sent_before_auth_rewrote_it(
+    monkeypatch, client_model, expected
+):
+    """LIT-3054: auth resolves router_settings.model_group_alias in the body, so the alias the
+    client sent only survives in the request scope. The response must still echo it."""
+    import litellm.proxy.common_request_processing as cpr
+
+    async def llm():
+        return litellm.ModelResponse(
+            model="gpt-4o-mini", choices=[{"message": {"role": "assistant", "content": "pong"}}]
+        )
+
+    async def fake_route_request(**_kwargs):
+        return llm()
+
+    logging_obj = MagicMock(litellm_call_id="call-id", _defer_async_logging=False)
+    proxy_logging = MagicMock(spec=ProxyLogging)
+    proxy_logging.during_call_hook = AsyncMock(return_value=None)
+    proxy_logging.post_call_success_hook = AsyncMock(side_effect=lambda data, user_api_key_dict, response: response)
+    proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={})
+    proxy_logging._callback_capabilities_cache = {}
+    monkeypatch.setattr(cpr, "route_request", fake_route_request)
+
+    processor = ProxyBaseLLMRequestProcessing(data={"model": "gpt-mini", "messages": []})
+    monkeypatch.setattr(
+        processor, "common_processing_pre_call_logic", AsyncMock(return_value=({"model": "gpt-mini"}, logging_obj))
+    )
+    monkeypatch.setattr(processor, "_has_post_call_guardrails", MagicMock(return_value=False))
+    scope = {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""}
+    request = Request({**scope, CLIENT_REQUESTED_MODEL_SCOPE_KEY: client_model} if client_model else scope)
+
+    response = await processor.base_process_llm_request(
+        request=request,
+        fastapi_response=Response(),
+        user_api_key_dict=ProxyUserAPIKeyAuth(),
+        proxy_logging_obj=proxy_logging,
+        general_settings={},
+        proxy_config=MagicMock(spec=ProxyConfig),
+        route_type="acompletion",
+        version=None,
+    )
+
+    assert response.model == expected
+
+
 class TestStreamingClientDisconnectLogging:
     @pytest.mark.asyncio
     async def test_record_streaming_client_disconnect_sets_error_information(self):

From 460a0128c94412890ed67588b6ad2de3347e766a Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:08:39 +0000
Subject: [PATCH 33/67] refactor(proxy): drop docstrings from key model limit
 helpers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/auth_utils.py                   | 6 ------
 litellm/proxy/hooks/parallel_request_limiter_v3.py | 2 --
 2 files changed, 8 deletions(-)

diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index 5ff85ad4a0e..b6c4a43149d 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -980,12 +980,6 @@ def get_key_own_model_rate_limit(
     user_api_key_dict: UserAPIKeyAuth,
     rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
 ) -> dict[str, int] | None:
-    """
-    Per-model limit the key sets on itself: key metadata first, then model_max_budget.
-
-    Unlike get_key_model_rpm_limit / get_key_model_tpm_limit this never falls back to the
-    team, so callers can tell a key override apart from an inherited team limit.
-    """
     if user_api_key_dict.metadata:
         result: Final = user_api_key_dict.metadata.get(rate_limit_key)
         if result:
diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py
index 87336830976..f35fb1e0042 100644
--- a/litellm/proxy/hooks/parallel_request_limiter_v3.py
+++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py
@@ -2899,7 +2899,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         requested_model: str,
         rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
     ) -> int | None:
-        """Team per-model limit this key inherits: None when the key sets its own limit for the model."""
         team_limits: Final = get_model_rate_limit_from_metadata(user_api_key_dict, "team_metadata", rate_limit_key)
         team_limit: Final = team_limits.get(requested_model) if team_limits else None
         if team_limit is None:
@@ -2915,7 +2914,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         requested_model: str | None,
         descriptors: list[RateLimitDescriptor],
     ) -> None:
-        """Add the team's per-model descriptor for the metrics the key does not override itself."""
         if requested_model is None:
             return
         team_rpm_limit: Final = self._inherited_team_model_limit(user_api_key_dict, requested_model, "model_rpm_limit")

From af4a0b4bc366f22d89b9bf20d5ec4feec523b9e0 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 21:50:07 +0000
Subject: [PATCH 34/67] fix(proxy): keep yaml pass-through endpoints visible to
 auth after db overlay

Resolves LIT-2053

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/proxy_server.py                 |  8 +++--
 tests/test_litellm/proxy/test_proxy_server.py | 34 +++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index f63e088ebf7..7f90874e38a 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -7080,8 +7080,12 @@ class ProxyConfig:
 
         ## PASS-THROUGH ENDPOINTS ##
         if "pass_through_endpoints" in _general_settings:
-            general_settings["pass_through_endpoints"] = _general_settings["pass_through_endpoints"]
-            await initialize_pass_through_endpoints(pass_through_endpoints=general_settings["pass_through_endpoints"])
+            db_pass_through_endpoints: Final = _general_settings["pass_through_endpoints"]
+            general_settings["pass_through_endpoints"] = [
+                *db_pass_through_endpoints,
+                *(config_passthrough_endpoints or []),
+            ]
+            await initialize_pass_through_endpoints(pass_through_endpoints=db_pass_through_endpoints)
 
         ## UI ACCESS MODE ##
         if "ui_access_mode" in _general_settings:
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 42af8e0af21..62c34738e51 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -7401,6 +7401,40 @@ async def test_update_general_settings_apply_user_budget_to_team_keys_yaml_wins(
         assert ps.general_settings["apply_user_budget_to_team_keys"] is True
 
 
+@pytest.mark.asyncio
+async def test_update_general_settings_keeps_yaml_pass_through_endpoints_next_to_db_ones():
+    """user_api_key_auth honours ``auth: false`` only for entries it finds in
+    general_settings["pass_through_endpoints"]. The DB overlay used to replace that
+    list wholesale, so once one endpoint existed in the DB the YAML-declared
+    auth-disabled route started answering 401 while staying registered."""
+    from litellm.proxy._types import ProxyException
+    from litellm.proxy.proxy_server import ProxyConfig
+
+    yaml_endpoint: Final = {"path": "/v1/cuopt/request", "target": "https://example.com/post", "auth": False}
+    db_endpoint: Final = {"id": "db-1", "path": "/v1/db-echo", "target": "https://example.com/post", "auth": True}
+
+    def request_without_key(path: str) -> MagicMock:
+        request: Final = MagicMock()
+        request.url.path = path
+        request.headers = {}
+        request.query_params = {}
+        return request
+
+    settings: Final = patch("litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [yaml_endpoint]})  # test-quality-ok: the method reads this module global; no injection seam
+    yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", [yaml_endpoint])  # test-quality-ok: module global holding the YAML endpoints the fix merges in
+    initialize: Final = patch("litellm.proxy.proxy_server.initialize_pass_through_endpoints", AsyncMock())  # test-quality-ok: route registration needs the FastAPI app; auth is the observable here
+    master_key: Final = patch("litellm.proxy.proxy_server.master_key", "sk-master")  # test-quality-ok: a set master key is what makes a missing Authorization header a 401
+    with settings, yaml_endpoints, initialize, master_key:
+        await ProxyConfig()._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]})
+
+        anonymous: Final = await user_api_key_auth(request=request_without_key("/v1/cuopt/request"), api_key=None)
+        assert anonymous.api_key is None
+
+        with pytest.raises(ProxyException) as still_protected:
+            await user_api_key_auth(request=request_without_key("/v1/db-echo"), api_key=None)
+        assert still_protected.value.code == "401"
+
+
 def _fill_user_api_key_cache(cache: DualCache, count: int) -> None:
     for index in range(count):
         cache.set_cache(key=f"key-{index}", value={"token": f"key-{index}"}, local_only=True)

From 69be041da7a8fc905862c58cf47cac714c402558 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:11:08 +0000
Subject: [PATCH 35/67] feat(proxy): add /nvidia_nim passthrough route for NIM
 object detection and OCR /v1/infer

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../azure_ai/passthrough/transformation.py    |  25 +-
 .../base_llm/passthrough/transformation.py    |  23 +-
 .../llms/nvidia_nim/passthrough/__init__.py   |   0
 .../nvidia_nim/passthrough/transformation.py  | 116 ++++++++++
 litellm/proxy/_types.py                       |   1 +
 litellm/proxy/auth/auth_utils.py              |  12 +
 .../llm_passthrough_endpoints.py              |  67 ++++++
 litellm/utils.py                              |   6 +
 ...t_nvidia_nim_passthrough_transformation.py | 216 ++++++++++++++++++
 .../proxy/auth/test_auth_utils.py             |  58 +++++
 .../proxy/auth/test_route_checks.py           |   1 +
 .../test_llm_pass_through_endpoints.py        | 147 ++++++++++++
 12 files changed, 649 insertions(+), 23 deletions(-)
 create mode 100644 litellm/llms/nvidia_nim/passthrough/__init__.py
 create mode 100644 litellm/llms/nvidia_nim/passthrough/transformation.py
 create mode 100644 tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py

diff --git a/litellm/llms/azure_ai/passthrough/transformation.py b/litellm/llms/azure_ai/passthrough/transformation.py
index f2be1d95593..4007ac37948 100644
--- a/litellm/llms/azure_ai/passthrough/transformation.py
+++ b/litellm/llms/azure_ai/passthrough/transformation.py
@@ -5,7 +5,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
 import httpx
-from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.llms.azure_ai.common_utils import (
@@ -18,6 +18,8 @@ from litellm.llms.base_llm.passthrough.transformation import (
     BasePassthroughConfig,
     RelayShape,
     logged_relay_shape,
+    model_group_from,
+    relayed_body,
     strip_leading_model_segment,
 )
 from litellm.types.llms.openai import AllMessageValues
@@ -35,19 +37,6 @@ if TYPE_CHECKING:
 EMPTY_QUERY: Final[Mapping[str, object]] = MappingProxyType({})
 
 
-class PassthroughMetadata(BaseModel):
-    model_config = ConfigDict(extra="ignore")
-
-    model_group: str = ""
-
-
-def model_group_from(litellm_params: Mapping[str, object]) -> str:
-    try:
-        return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group
-    except ValidationError:
-        return ""
-
-
 def api_version_from(litellm_params: Mapping[str, object]) -> str | None:
     try:
         return TypeAdapter(str | None).validate_python(litellm_params.get("api_version"))
@@ -96,14 +85,6 @@ def relay_query_params(
     return MappingProxyType({**(request_query_params or EMPTY_QUERY), "api-version": api_version})
 
 
-def relayed_body(httpx_response: Response) -> str | dict:
-    try:
-        body: Final[object] = httpx_response.json()
-    except ValueError:
-        return httpx_response.text
-    return body if isinstance(body, dict) else httpx_response.text
-
-
 FOUNDRY_RELAY_SHAPES: Final = (
     RelayShape("/rerank", CallTypes.arerank, RerankResponse.model_validate),
     RelayShape("/providers/blackforestlabs/v1/flux-2-pro", CallTypes.aimage_generation, ImageResponse.model_validate),
diff --git a/litellm/llms/base_llm/passthrough/transformation.py b/litellm/llms/base_llm/passthrough/transformation.py
index ec938889b88..f2a12c3f22d 100644
--- a/litellm/llms/base_llm/passthrough/transformation.py
+++ b/litellm/llms/base_llm/passthrough/transformation.py
@@ -6,7 +6,7 @@ from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
 
-from pydantic import TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm.types.utils import CallTypes
 
@@ -29,6 +29,19 @@ if TYPE_CHECKING:
 RELAYED_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
 
 
+class PassthroughMetadata(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    model_group: str = ""
+
+
+def model_group_from(litellm_params: Mapping[str, object]) -> str:
+    try:
+        return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group
+    except ValidationError:
+        return ""
+
+
 def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str:
     path: Final = endpoint.lstrip("/")
     for model_name in model_names:
@@ -55,6 +68,14 @@ def relayed_json_object(httpx_response: Response) -> Mapping[str, object] | None
         return None
 
 
+def relayed_body(httpx_response: Response) -> str | dict:
+    try:
+        body: Final[object] = httpx_response.json()
+    except ValueError:
+        return httpx_response.text
+    return body if isinstance(body, dict) else httpx_response.text
+
+
 @dataclass(frozen=True, slots=True)
 class RelayShape:
     path_suffix: str
diff --git a/litellm/llms/nvidia_nim/passthrough/__init__.py b/litellm/llms/nvidia_nim/passthrough/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/llms/nvidia_nim/passthrough/transformation.py b/litellm/llms/nvidia_nim/passthrough/transformation.py
new file mode 100644
index 00000000000..f6273daaabb
--- /dev/null
+++ b/litellm/llms/nvidia_nim/passthrough/transformation.py
@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Collection, Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+import httpx
+
+from litellm.llms.base_llm.passthrough.transformation import (
+    BasePassthroughConfig,
+    model_group_from,
+    relayed_body,
+    strip_leading_model_segment,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import StandardPassThroughResponseObject
+
+if TYPE_CHECKING:
+    from httpx import URL, Response
+
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    from litellm.llms.base_llm.ocr.transformation import OCRResponse
+    from litellm.llms.base_llm.passthrough.transformation import LoggedRelayResponse
+
+
+API_VERSION_SEGMENT: Final = re.compile(r"^v\d+$")
+
+
+def nvidia_nim_router_model_in_endpoint(endpoint: str, router_models: Collection[str]) -> str | None:
+    segments: Final = tuple(segment for segment in endpoint.split("/") if segment)
+    return next(
+        (
+            "/".join(segments[:length])
+            for length in range(len(segments), 0, -1)
+            if "/".join(segments[:length]) in router_models
+        ),
+        None,
+    )
+
+
+def without_repeated_version_prefix(api_base: str, native_endpoint: str) -> str:
+    url: Final = httpx.URL(api_base)
+    base_segments: Final = tuple(segment for segment in url.path.split("/") if segment)
+    first_native_segment: Final = native_endpoint.lstrip("/").split("/", 1)[0]
+    repeated: Final = (
+        bool(base_segments)
+        and API_VERSION_SEGMENT.match(first_native_segment) is not None
+        and base_segments[-1] == first_native_segment
+    )
+    kept_segments: Final = base_segments[:-1] if repeated else base_segments
+    return str(url.copy_with(path="/" + "/".join(kept_segments), query=None)).rstrip("/")
+
+
+class NvidiaNimPassthroughConfig(BasePassthroughConfig):
+    def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
+        return bool(request_data.get("stream", False))
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        endpoint: str,
+        request_query_params: dict | None,
+        litellm_params: dict,
+    ) -> tuple[URL, str]:
+        base_target_url: Final = self.get_api_base(api_base)
+        if base_target_url is None:
+            raise ValueError("NVIDIA NIM api base not found: set `api_base` on the deployment or NVIDIA_NIM_API_BASE")
+        native_endpoint: Final = strip_leading_model_segment(endpoint, (model_group_from(litellm_params), model))
+        root: Final = without_repeated_version_prefix(base_target_url, native_endpoint)
+        return (self.format_url(native_endpoint, root, request_query_params), root)
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: base class contract returns dict for httpx
+        if api_key is None:
+            return dict(headers)  # mutable-ok: base class contract returns dict for httpx
+        return {
+            **headers,
+            "Authorization": f"Bearer {api_key}",
+        }  # mutable-ok: base class contract returns dict for httpx
+
+    @staticmethod
+    def get_api_base(api_base: str | None = None) -> str | None:
+        return api_base or get_secret_str("NVIDIA_NIM_API_BASE")
+
+    @staticmethod
+    def get_api_key(api_key: str | None = None) -> str | None:
+        return api_key or get_secret_str("NVIDIA_NIM_API_KEY")
+
+    @staticmethod
+    def get_base_model(model: str) -> str | None:
+        return model
+
+    def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
+        return []
+
+    def logging_non_streaming_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        httpx_response: Response,
+        request_data: Mapping[str, object],
+        logging_obj: Logging,
+        endpoint: str,
+    ) -> LoggedRelayResponse | OCRResponse | StandardPassThroughResponseObject | None:
+        return StandardPassThroughResponseObject(response=relayed_body(httpx_response))
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index d4eda1c9540..b780b8d08ca 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -485,6 +485,7 @@ class LiteLLMRoutes(enum.Enum):
         "/milvus",
         "/gigachat",
         "/watsonx",
+        "/nvidia_nim",
     ]
 
     #########################################################
diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index dc304a156cf..529521dcba8 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -28,6 +28,7 @@ from litellm.litellm_core_utils.url_utils import (
     validate_url,
 )
 from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
+from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_router_model_in_endpoint
 from litellm.proxy._types import *
 from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
@@ -2040,9 +2041,20 @@ def get_model_from_request(
         azure_model: Final = _router_model_from_azure_route(route, llm_router)
         return model if azure_model is None else azure_model
 
+    if route.lower().startswith("/nvidia_nim/"):
+        nvidia_nim_model: Final = _router_model_from_nvidia_nim_route(route, llm_router)
+        return model if nvidia_nim_model is None else nvidia_nim_model
+
     return model
 
 
+def _router_model_from_nvidia_nim_route(route: str, llm_router: Router | None) -> str | None:
+    if llm_router is None:
+        return None
+    endpoint: Final = re.sub(r"^/nvidia_nim/", "", route, flags=re.IGNORECASE)
+    return nvidia_nim_router_model_in_endpoint(endpoint, frozenset(llm_router.get_model_names()))
+
+
 def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None:
     if llm_router is None:
         return None
diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index 28a8bab1f24..d4b1bdbdc20 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -36,6 +36,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.azure.passthrough.transformation import foreign_azure_deployment
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_router_model_in_endpoint
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.passthrough.main import AsyncPassthroughStreamingResponse
 from litellm.proxy._types import *
@@ -1545,6 +1546,26 @@ async def _relay_azure_router_model(
             "put the model group name in the deployments segment"
         }
         raise HTTPException(status_code=400, detail=rejection)
+    return await _relay_router_model(
+        llm_router=llm_router,
+        model=model,
+        endpoint=endpoint,
+        request=request,
+        request_body=request_body,
+        is_streaming_request=is_streaming_request,
+        user_api_key_dict=user_api_key_dict,
+    )
+
+
+async def _relay_router_model(
+    llm_router: litellm.Router,
+    model: str,
+    endpoint: str,
+    request: Request,
+    request_body: Mapping[str, object],
+    is_streaming_request: bool,
+    user_api_key_dict: UserAPIKeyAuth,
+) -> Response:
     try:
         result: Final = await llm_router.allm_passthrough_route(
             model=model,
@@ -1594,6 +1615,52 @@ async def _relay_azure_router_model(
     )
 
 
+@router.api_route(
+    "/nvidia_nim/{endpoint:path}",
+    methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+    tags=["NVIDIA NIM Pass-through", "pass-through"],
+)
+async def nvidia_nim_proxy_route(
+    endpoint: str,
+    request: Request,
+    fastapi_response: Response,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+):
+    """
+    Relay a native NVIDIA NIM request through a LiteLLM model group.
+
+    `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's
+    `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through
+    virtual key auth, model access checks, and spend logging.
+    """
+    from litellm.proxy.proxy_server import llm_router
+
+    model_group: Final = (
+        nvidia_nim_router_model_in_endpoint(endpoint, llm_router.get_model_names()) if llm_router else None
+    )
+    if llm_router is None or model_group is None:
+        rejection: Final[RelayRejection] = {
+            "error": "no LiteLLM model group in the path; call /nvidia_nim/{model_group}/v1/infer with a model "
+            "from your `model_list` whose `model` starts with `nvidia_nim/`"
+        }
+        raise HTTPException(status_code=400, detail=rejection)
+
+    request_body: Final = await get_request_body(request)
+    is_streaming_request: Final = is_passthrough_request_streaming(request_body)
+    return await open_sse_before_first_byte(
+        _relay_router_model(
+            llm_router=llm_router,
+            model=model_group,
+            endpoint=endpoint,
+            request=request,
+            request_body=request_body,
+            is_streaming_request=is_streaming_request,
+            user_api_key_dict=user_api_key_dict,
+        ),
+        ping_interval_seconds=(litellm.sse_keepalive_ping_interval_seconds if is_streaming_request else None),
+    )
+
+
 @router.api_route(
     "/azure_ai/{endpoint:path}",
     methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
diff --git a/litellm/utils.py b/litellm/utils.py
index af22b11224b..7f151e1d581 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -8989,6 +8989,12 @@ class ProviderConfigManager:
             )
 
             return WatsonxPassthroughConfig()
+        elif LlmProviders.NVIDIA_NIM == provider:
+            from litellm.llms.nvidia_nim.passthrough.transformation import (
+                NvidiaNimPassthroughConfig,
+            )
+
+            return NvidiaNimPassthroughConfig()
         return None
 
     @staticmethod
diff --git a/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py b/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py
new file mode 100644
index 00000000000..87de4a795e0
--- /dev/null
+++ b/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py
@@ -0,0 +1,216 @@
+import json
+from types import MappingProxyType
+
+import httpx
+import pytest
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.llms.nvidia_nim.passthrough.transformation import (
+    NvidiaNimPassthroughConfig,
+    nvidia_nim_router_model_in_endpoint,
+)
+from litellm.types.utils import LlmProviders
+from litellm.utils import ProviderConfigManager
+
+NIM_BASE = "http://nim.internal:8000"
+INFER_BODY = {
+    "input": [
+        {"type": "image_url", "url": "data:image/png;base64,AAAA"},
+        {"type": "image_url", "url": "data:image/png;base64,BBBB"},
+    ]
+}
+
+
+@pytest.fixture(autouse=True)
+def clear_nvidia_nim_env(monkeypatch):
+    for env_var in ("NVIDIA_NIM_API_BASE", "NVIDIA_NIM_API_KEY"):
+        monkeypatch.delenv(env_var, raising=False)
+    monkeypatch.setattr(litellm, "api_base", None)
+    monkeypatch.setattr(litellm, "api_key", None)
+
+
+def test_provider_config_manager_resolves_nvidia_nim_passthrough_config():
+    config = ProviderConfigManager.get_provider_passthrough_config(
+        model="nvidia/nemoretriever-page-elements-v2", provider=LlmProviders.NVIDIA_NIM
+    )
+
+    assert isinstance(config, NvidiaNimPassthroughConfig)
+
+
+@pytest.mark.parametrize(
+    "api_base, endpoint, litellm_params, expected",
+    [
+        (NIM_BASE, "nim-page/v1/infer", {"litellm_metadata": {"model_group": "nim-page"}}, f"{NIM_BASE}/v1/infer"),
+        (
+            f"{NIM_BASE}/v1",
+            "nim-page/v1/infer",
+            {"litellm_metadata": {"model_group": "nim-page"}},
+            f"{NIM_BASE}/v1/infer",
+        ),
+        (f"{NIM_BASE}/v1/", "/v1/infer", {}, f"{NIM_BASE}/v1/infer"),
+        (NIM_BASE, "v1/infer", {}, f"{NIM_BASE}/v1/infer"),
+        (f"{NIM_BASE}/v2", "v1/infer", {}, f"{NIM_BASE}/v2/v1/infer"),
+        (f"{NIM_BASE}/infer", "infer", {}, f"{NIM_BASE}/infer/infer"),
+        (NIM_BASE, "nvidia/nemoretriever-page-elements-v2/v1/infer", {}, f"{NIM_BASE}/v1/infer"),
+    ],
+)
+def test_relay_url_strips_the_model_group_and_never_doubles_the_api_version(
+    api_base, endpoint, litellm_params, expected
+):
+    url, base = NvidiaNimPassthroughConfig().get_complete_url(
+        api_base=api_base,
+        api_key=None,
+        model="nvidia/nemoretriever-page-elements-v2",
+        endpoint=endpoint,
+        request_query_params=None,
+        litellm_params=litellm_params,
+    )
+
+    assert str(url) == expected
+    assert base == expected.removesuffix("/v1/infer").removesuffix("/infer")
+
+
+def test_query_params_are_forwarded_on_the_relay_url():
+    url, _ = NvidiaNimPassthroughConfig().get_complete_url(
+        api_base=NIM_BASE,
+        api_key=None,
+        model="nvidia/nemoretriever-page-elements-v2",
+        endpoint="v1/infer",
+        request_query_params={"timeout": "30"},
+        litellm_params={},
+    )
+
+    assert str(url) == f"{NIM_BASE}/v1/infer?timeout=30"
+
+
+def test_env_api_base_is_used_when_the_deployment_has_none(monkeypatch):
+    monkeypatch.setenv("NVIDIA_NIM_API_BASE", f"{NIM_BASE}/v1")
+
+    url, _ = NvidiaNimPassthroughConfig().get_complete_url(
+        api_base=None,
+        api_key=None,
+        model="nvidia/nemoretriever-page-elements-v2",
+        endpoint="v1/infer",
+        request_query_params=None,
+        litellm_params={},
+    )
+
+    assert str(url) == f"{NIM_BASE}/v1/infer"
+
+
+def test_missing_api_base_raises_instead_of_building_a_relative_url():
+    with pytest.raises(ValueError, match="NVIDIA_NIM_API_BASE"):
+        NvidiaNimPassthroughConfig().get_complete_url(
+            api_base=None,
+            api_key=None,
+            model="nvidia/nemoretriever-page-elements-v2",
+            endpoint="v1/infer",
+            request_query_params=None,
+            litellm_params={},
+        )
+
+
+def test_deployment_key_becomes_a_bearer_token_and_caller_headers_are_kept():
+    caller_headers = MappingProxyType({"x-request-id": "abc"})
+
+    headers = NvidiaNimPassthroughConfig().validate_environment(
+        headers=caller_headers,
+        model="nvidia/nemoretriever-page-elements-v2",
+        messages=[],
+        optional_params={},
+        litellm_params={},
+        api_key="nvapi-secret",
+    )
+
+    assert headers == {"x-request-id": "abc", "Authorization": "Bearer nvapi-secret"}
+
+
+def test_self_hosted_nim_without_a_key_sends_no_authorization_header():
+    headers = NvidiaNimPassthroughConfig().validate_environment(
+        headers={}, model="nvidia/x", messages=[], optional_params={}, litellm_params={}, api_key=None
+    )
+
+    assert "Authorization" not in headers
+
+
+def test_env_api_key_fills_in_when_the_deployment_has_none(monkeypatch):
+    monkeypatch.setenv("NVIDIA_NIM_API_KEY", "nvapi-from-env")
+
+    assert NvidiaNimPassthroughConfig.get_api_key(None) == "nvapi-from-env"
+    assert NvidiaNimPassthroughConfig.get_api_key("nvapi-deployment") == "nvapi-deployment"
+
+
+@pytest.mark.parametrize(
+    "endpoint, router_models, expected",
+    [
+        ("nim-page/v1/infer", ("nim-page", "nim-table"), "nim-page"),
+        ("/nim-page/v1/infer", ("nim-page",), "nim-page"),
+        (
+            "nvidia/nemoretriever-page-elements-v2/v1/infer",
+            ("nvidia/nemoretriever-page-elements-v2",),
+            "nvidia/nemoretriever-page-elements-v2",
+        ),
+        ("nim/v1/infer", ("nim", "nim/v1"), "nim/v1"),
+        ("v1/infer", ("nim-page",), None),
+        ("nim-page-elements/v1/infer", ("nim-page",), None),
+        ("", ("nim-page",), None),
+    ],
+)
+def test_router_model_in_endpoint_takes_the_longest_leading_model_group(endpoint, router_models, expected):
+    assert nvidia_nim_router_model_in_endpoint(endpoint, frozenset(router_models)) == expected
+
+
+@pytest.mark.parametrize("request_data, expected", [({"stream": True}, True), ({"stream": False}, False), ({}, False)])
+def test_is_streaming_request_reads_the_stream_flag(request_data, expected):
+    assert NvidiaNimPassthroughConfig().is_streaming_request("v1/infer", request_data) is expected
+
+
+def test_non_streaming_relay_logs_the_upstream_json_body():
+    response = httpx.Response(
+        200,
+        json={"data": [{"index": 0, "bounding_boxes": {}}]},
+        request=httpx.Request("POST", f"{NIM_BASE}/v1/infer"),
+    )
+
+    result = NvidiaNimPassthroughConfig().logging_non_streaming_response(
+        model="nvidia/nemoretriever-page-elements-v2",
+        custom_llm_provider="nvidia_nim",
+        httpx_response=response,
+        request_data=INFER_BODY,
+        logging_obj=None,  # pyright: ignore[reportArgumentType]  # not read for a plain passthrough body
+        endpoint="v1/infer",
+    )
+
+    assert result == {"response": {"data": [{"index": 0, "bounding_boxes": {}}]}}
+
+
+@pytest.mark.asyncio
+async def test_object_detection_relay_sends_the_native_body_unchanged_to_v1_infer():
+    upstream_requests: list[httpx.Request] = []
+
+    def nim(request: httpx.Request) -> httpx.Response:
+        upstream_requests.append(request)
+        return httpx.Response(200, json={"data": [{"index": 0}, {"index": 1}]}, headers={"x-nim": "1"})
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(nim))
+
+    response = await litellm.allm_passthrough_route(
+        model="nvidia_nim/nvidia/nemoretriever-page-elements-v2",
+        endpoint="nim-page/v1/infer",
+        method="POST",
+        api_base=f"{NIM_BASE}/v1",
+        api_key="nvapi-secret",
+        json=dict(INFER_BODY),
+        litellm_metadata={"model_group": "nim-page"},
+        client=client,
+    )
+
+    (sent,) = upstream_requests
+    assert str(sent.url) == f"{NIM_BASE}/v1/infer"
+    assert json.loads(sent.content) == INFER_BODY
+    assert sent.headers["authorization"] == "Bearer nvapi-secret"
+    assert response.status_code == 200
+    assert response.headers["x-nim"] == "1"
+    assert response.json() == {"data": [{"index": 0}, {"index": 1}]}
diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py
index bd6a14cad21..81e505ea0a9 100644
--- a/tests/test_litellm/proxy/auth/test_auth_utils.py
+++ b/tests/test_litellm/proxy/auth/test_auth_utils.py
@@ -823,6 +823,64 @@ def test_get_model_from_request_azure_relay_routes_use_the_model_group_in_the_pa
     assert get_model_from_request(request_data=request_data, route=route, llm_router=_azure_relay_router()) == expected
 
 
+def _nvidia_nim_relay_router():
+    from litellm.router import Router
+
+    return Router(
+        model_list=[
+            {
+                "model_name": "nim-page-elements",
+                "litellm_params": {
+                    "model": "nvidia_nim/nvidia/nemoretriever-page-elements-v2",
+                    "api_base": "http://nim-a.internal:8000",
+                    "api_key": "k",
+                },
+            },
+            {
+                "model_name": "nvidia/nemoretriever-table-structure-v1",
+                "litellm_params": {
+                    "model": "nvidia_nim/nvidia/nemoretriever-table-structure-v1",
+                    "api_base": "http://nim-b.internal:8000",
+                    "api_key": "k",
+                },
+            },
+        ]
+    )
+
+
+NIM_INFER_BODY = {"input": [{"type": "image_url", "url": "data:image/png;base64,AAAA"}]}
+
+
+@pytest.mark.parametrize(
+    "route, request_data, expected",
+    [
+        ("/nvidia_nim/nim-page-elements/v1/infer", NIM_INFER_BODY, "nim-page-elements"),
+        (
+            "/nvidia_nim/nim-page-elements/v1/infer",
+            {"model": "nvidia/nemoretriever-table-structure-v1"},
+            "nim-page-elements",
+        ),
+        (
+            "/nvidia_nim/nvidia/nemoretriever-table-structure-v1/v1/infer",
+            NIM_INFER_BODY,
+            "nvidia/nemoretriever-table-structure-v1",
+        ),
+        ("/nvidia_nim/v1/infer", NIM_INFER_BODY, None),
+        ("/nvidia_nim/unknown-group/v1/infer", NIM_INFER_BODY, None),
+        ("/nvidia_nim/nim-page-elements-v2/v1/infer", NIM_INFER_BODY, None),
+    ],
+)
+def test_get_model_from_request_nvidia_nim_relay_routes_use_the_model_group_in_the_path(route, request_data, expected):
+    assert (
+        get_model_from_request(request_data=request_data, route=route, llm_router=_nvidia_nim_relay_router())
+        == expected
+    )
+
+
+def test_get_model_from_request_nvidia_nim_relay_without_a_router_has_no_model():
+    assert get_model_from_request(request_data=NIM_INFER_BODY, route="/nvidia_nim/nim-page-elements/v1/infer") is None
+
+
 def test_get_model_from_request_includes_file_endpoint_header_model():
     assert (
         get_model_from_request(
diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py
index 0950b56bf03..806c55d51ce 100644
--- a/tests/test_litellm/proxy/auth/test_route_checks.py
+++ b/tests/test_litellm/proxy/auth/test_route_checks.py
@@ -693,6 +693,7 @@ def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied():
         "/anthropic/v1/count_tokens",
         "/gemini/v1/models",
         "/gemini/countTokens",
+        "/nvidia_nim/nim-page-elements/v1/infer",
     ],
 )
 def test_virtual_key_llm_api_route_includes_passthrough_prefix(route):
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
index 7b285674145..4ef04a8e633 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
@@ -40,6 +40,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     llm_passthrough_factory_proxy_route,
     milvus_proxy_route,
     mistral_proxy_route,
+    nvidia_nim_proxy_route,
     openai_proxy_route,
     vertex_discovery_proxy_route,
     vertex_proxy_route,
@@ -5375,6 +5376,152 @@ class TestRouterModelRelayUpstreamContract:
         assert result.headers["x-ms-request-id"] == "req-1"
 
 
+NIM_INFER_BODY = {
+    "input": [
+        {"type": "image_url", "url": "data:image/png;base64,AAAA"},
+        {"type": "image_url", "url": "data:image/png;base64,BBBB"},
+    ]
+}
+
+
+class TestNvidiaNimProxyRoute:
+    def _request(self) -> MagicMock:
+        request = MagicMock(spec=Request)
+        request.method = "POST"
+        request.headers = {"content-type": "application/json"}
+        request.query_params = {}
+        return request
+
+    def _install_router(self, monkeypatch, router, body: dict) -> None:
+        import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep
+        import litellm.proxy.proxy_server as proxy_server
+
+        async def fake_get_request_body(_request):
+            return dict(body)
+
+        monkeypatch.setattr(proxy_server, "llm_router", router)
+        monkeypatch.setattr(ep, "get_request_body", fake_get_request_body)
+
+    def _recording_router(self, captured: list[dict], model_names: tuple[str, ...]):
+        class RecordingRouter:
+            def get_model_names(self):
+                return list(model_names)
+
+            async def allm_passthrough_route(self, **kwargs):
+                captured.append(kwargs)
+                return httpx.Response(
+                    200, json={"data": [{"index": 0, "bounding_boxes": {}}]}, headers={"x-nim-request": "r1"}
+                )
+
+        return RecordingRouter()
+
+    @pytest.mark.asyncio
+    async def test_model_group_in_the_path_selects_the_deployment_and_the_body_stays_model_free(self, monkeypatch):
+        captured: list[dict] = []
+        self._install_router(
+            monkeypatch, self._recording_router(captured, ("nim-page-elements", "nim-table")), NIM_INFER_BODY
+        )
+
+        result = await nvidia_nim_proxy_route(
+            endpoint="nim-page-elements/v1/infer",
+            request=self._request(),
+            fastapi_response=MagicMock(spec=Response),
+            user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token", team_id="team-1"),
+        )
+
+        (relay,) = captured
+        assert relay["model"] == "nim-page-elements"
+        assert relay["endpoint"] == "nim-page-elements/v1/infer"
+        assert relay["method"] == "POST"
+        assert relay["json"] == NIM_INFER_BODY
+        assert "model" not in relay["json"]
+        assert relay["litellm_metadata"]["user_api_key_team_id"] == "team-1"
+        assert result.status_code == 200
+        assert json.loads(result.body) == {"data": [{"index": 0, "bounding_boxes": {}}]}
+        assert result.headers["x-nim-request"] == "r1"
+
+    @pytest.mark.asyncio
+    async def test_model_group_with_a_slash_is_matched_as_the_longest_leading_path(self, monkeypatch):
+        captured: list[dict] = []
+        self._install_router(
+            monkeypatch, self._recording_router(captured, ("nvidia/nemoretriever-page-elements-v2",)), NIM_INFER_BODY
+        )
+
+        await nvidia_nim_proxy_route(
+            endpoint="nvidia/nemoretriever-page-elements-v2/v1/infer",
+            request=self._request(),
+            fastapi_response=MagicMock(spec=Response),
+            user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
+        )
+
+        assert captured[0]["model"] == "nvidia/nemoretriever-page-elements-v2"
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("endpoint", ["v1/infer", "unknown-group/v1/infer", "nim-page-elements-v2/v1/infer"])
+    async def test_path_without_a_configured_model_group_is_rejected_before_any_upstream_call(
+        self, monkeypatch, endpoint
+    ):
+        from fastapi import HTTPException
+
+        captured: list[dict] = []
+        self._install_router(monkeypatch, self._recording_router(captured, ("nim-page-elements",)), NIM_INFER_BODY)
+
+        with pytest.raises(HTTPException) as exc_info:
+            await nvidia_nim_proxy_route(
+                endpoint=endpoint,
+                request=self._request(),
+                fastapi_response=MagicMock(spec=Response),
+                user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
+            )
+
+        assert exc_info.value.status_code == 400
+        assert captured == []
+
+    @pytest.mark.asyncio
+    async def test_no_router_is_rejected_before_any_upstream_call(self, monkeypatch):
+        from fastapi import HTTPException
+
+        self._install_router(monkeypatch, None, NIM_INFER_BODY)
+
+        with pytest.raises(HTTPException) as exc_info:
+            await nvidia_nim_proxy_route(
+                endpoint="nim-page-elements/v1/infer",
+                request=self._request(),
+                fastapi_response=MagicMock(spec=Response),
+                user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
+            )
+
+        assert exc_info.value.status_code == 400
+
+    @pytest.mark.asyncio
+    async def test_upstream_rejection_is_relayed_with_its_status_body_and_headers(self, monkeypatch):
+        upstream_body = {"detail": "input[0].url must be a data URL"}
+
+        class RejectingRouter:
+            def get_model_names(self):
+                return ["nim-page-elements"]
+
+            async def allm_passthrough_route(self, **kwargs):
+                upstream_request = httpx.Request("POST", "http://nim.internal:8000/v1/infer")
+                upstream = httpx.Response(
+                    422, json=upstream_body, headers={"x-nim-request": "r2"}, request=upstream_request
+                )
+                raise httpx.HTTPStatusError("422", request=upstream_request, response=upstream)
+
+        self._install_router(monkeypatch, RejectingRouter(), {"input": [{"type": "image_url", "url": "x"}]})
+
+        result = await nvidia_nim_proxy_route(
+            endpoint="nim-page-elements/v1/infer",
+            request=self._request(),
+            fastapi_response=MagicMock(spec=Response),
+            user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
+        )
+
+        assert result.status_code == 422
+        assert json.loads(result.body) == upstream_body
+        assert result.headers["x-nim-request"] == "r2"
+
+
 @pytest.mark.asyncio
 async def test_bedrock_count_tokens_error_forwards_provider_headers():
     """The count tokens route converts BedrockError into an HTTPException, and dropping the

From c05af7a12f1eb68c0a4cb287c099a0ad6f684f54 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:12:06 +0000
Subject: [PATCH 36/67] feat(ui): add custom request headers to the API
 Playground

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../chat_ui/ChatUI.integration.test.tsx       | 63 ++++++++++++++++++-
 .../playground/components/chat_ui/ChatUI.tsx  | 45 ++++++++++---
 .../playground/llm_calls/a2a_send_message.tsx |  3 +
 .../llm_calls/anthropic_messages.test.tsx     | 23 +++++++
 .../llm_calls/anthropic_messages.tsx          |  8 +--
 .../playground/llm_calls/audio_speech.tsx     |  4 +-
 .../llm_calls/audio_transcriptions.tsx        |  4 +-
 .../llm_calls/embeddings_api.test.tsx         | 20 ++++++
 .../playground/llm_calls/embeddings_api.tsx   |  8 +--
 .../playground/llm_calls/image_edits.tsx      |  4 +-
 .../playground/llm_calls/image_generation.tsx |  4 +-
 .../playground/llm_calls/interactions_api.tsx |  6 +-
 .../components/chat_ui/CodeSnippets.test.tsx  | 22 +++++++
 .../src/components/chat_ui/CodeSnippets.tsx   | 12 +++-
 .../llm_calls/chat_completion.test.tsx        | 45 +++++++++++++
 .../components/llm_calls/chat_completion.tsx  |  8 +--
 .../llm_calls/request_headers.test.ts         | 44 +++++++++++++
 .../components/llm_calls/request_headers.ts   | 27 ++++++++
 .../llm_calls/responses_api.test.tsx          | 44 +++++++++++++
 .../components/llm_calls/responses_api.tsx    |  8 +--
 20 files changed, 364 insertions(+), 38 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts
 create mode 100644 ui/litellm-dashboard/src/components/llm_calls/request_headers.ts

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx
index 984996351df..e79d382ae39 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.integration.test.tsx
@@ -35,10 +35,12 @@ beforeEach(() => {
   Element.prototype.scrollIntoView = () => {};
 });
 
-const CHAT_REQUEST_ARG_COUNT = 26;
+const CHAT_REQUEST_ARG_COUNT = 27;
 const STREAMING_ENABLED_ARG_INDEX = 25;
-const MESSAGES_REQUEST_ARG_COUNT = 19;
+const CHAT_CUSTOM_HEADERS_ARG_INDEX = 26;
+const MESSAGES_REQUEST_ARG_COUNT = 20;
 const MESSAGES_STREAMING_ENABLED_ARG_INDEX = 18;
+const MESSAGES_CUSTOM_HEADERS_ARG_INDEX = 19;
 
 async function openComboboxByPlaceholder(placeholder: string) {
   const user = userEvent.setup();
@@ -447,6 +449,63 @@ describe("ChatUI", () => {
     expect(requestArgs[MESSAGES_STREAMING_ENABLED_ARG_INDEX]).toBe(false);
   });
 
+  it("should send custom headers entered in the sidebar with /v1/chat/completions and /v1/messages requests", async () => {
+    const user = userEvent.setup();
+    render(
+      <ChatUI
+        accessToken="1234567890"
+        token="1234567890"
+        userRole="user"
+        userID="1234567890"
+        disabledPersonalKeyCreation={false}
+      />,
+    );
+
+    await waitFor(() => {
+      expect(screen.getByText("Test Key")).toBeInTheDocument();
+    });
+
+    await selectComboboxOption("Select a Model", "Model 1");
+    await user.click(screen.getByRole("button", { name: "Add Header" }));
+    await user.click(screen.getByRole("button", { name: "Add Header" }));
+    const [firstName] = screen.getAllByPlaceholderText("Header Name");
+    const [firstValue, secondValue] = screen.getAllByPlaceholderText("Header Value");
+    fireEvent.change(firstName, { target: { value: "anthropic-beta" } });
+    fireEvent.change(firstValue, { target: { value: "context-1m-2025-08-07" } });
+    fireEvent.change(secondValue, { target: { value: "ignored because the name is blank" } });
+
+    const messageInput = screen.getByPlaceholderText("Type your message... (Shift+Enter for new line)");
+    await act(async () => {
+      fireEvent.change(messageInput, { target: { value: "hello" } });
+    });
+    await act(async () => {
+      fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" });
+    });
+
+    await waitFor(() => {
+      expect(makeOpenAIChatCompletionRequest).toHaveBeenCalledTimes(1);
+    });
+    const chatArgs = vi.mocked(makeOpenAIChatCompletionRequest).mock.calls[0];
+    expect(chatArgs).toHaveLength(CHAT_REQUEST_ARG_COUNT);
+    expect(chatArgs[CHAT_CUSTOM_HEADERS_ARG_INDEX]).toEqual({ "anthropic-beta": "context-1m-2025-08-07" });
+
+    await selectComboboxOption("Select an endpoint", "/v1/messages");
+    await selectComboboxOption("Select a Model", "Model 1");
+    await act(async () => {
+      fireEvent.change(messageInput, { target: { value: "hello again" } });
+    });
+    await act(async () => {
+      fireEvent.keyDown(messageInput, { key: "Enter", code: "Enter" });
+    });
+
+    await waitFor(() => {
+      expect(makeAnthropicMessagesRequest).toHaveBeenCalledTimes(1);
+    });
+    const messagesArgs = vi.mocked(makeAnthropicMessagesRequest).mock.calls[0];
+    expect(messagesArgs).toHaveLength(MESSAGES_REQUEST_ARG_COUNT);
+    expect(messagesArgs[MESSAGES_CUSTOM_HEADERS_ARG_INDEX]).toEqual({ "anthropic-beta": "context-1m-2025-08-07" });
+  });
+
   it("should force streaming in simplified mode even when the playground setting is off", async () => {
     sessionStorage.setItem("streamingEnabled", "false");
 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx
index ed8679cfdc1..ae0fabe5ef2 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx
@@ -9,6 +9,7 @@ import {
   Info,
   Key,
   Link2,
+  ListPlus,
   Loader2,
   Settings,
   Shield,
@@ -40,6 +41,8 @@ import { makeAnthropicMessagesRequest } from "../../llm_calls/anthropic_messages
 import { makeOpenAIAudioSpeechRequest } from "../../llm_calls/audio_speech";
 import { makeOpenAIAudioTranscriptionRequest } from "../../llm_calls/audio_transcriptions";
 import { makeOpenAIChatCompletionRequest } from "@/components/llm_calls/chat_completion";
+import { customHeadersFromPairs, parseStoredHeaderPairs } from "@/components/llm_calls/request_headers";
+import KeyValueInput, { type KeyValuePair } from "@/components/key_value_input";
 import { makeOpenAIEmbeddingsRequest } from "../../llm_calls/embeddings_api";
 import { Agent, fetchAvailableAgents } from "../../llm_calls/fetch_agents";
 import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
@@ -220,6 +223,10 @@ const ChatUI: React.FC<ChatUIProps> = ({
       return [];
     }
   });
+  const [customHeaderPairs, setCustomHeaderPairs] = useState<readonly KeyValuePair[]>(() =>
+    parseStoredHeaderPairs(getSecureItem("customHeaders")),
+  );
+  const customHeaders = useMemo(() => customHeadersFromPairs(customHeaderPairs), [customHeaderPairs]);
   const [selectedVoice, setSelectedVoice] = useState<OpenAIVoice>(() => {
     const saved = sessionStorage.getItem("selectedVoice");
     if (!saved) return "alloy";
@@ -346,6 +353,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
         selectedSdk,
         selectedVoice,
         proxySettings,
+        customHeaders,
       });
       setGeneratedCode(code);
     }
@@ -367,12 +375,14 @@ const ChatUI: React.FC<ChatUIProps> = ({
     endpointType,
     selectedModel,
     proxySettings,
+    customHeaders,
   ]);
 
   useEffect(() => {
     try {
       setSecureItem("apiKeySource", JSON.stringify(apiKeySource));
       setSecureItem("apiKey", apiKey);
+      setSecureItem("customHeaders", JSON.stringify(customHeaderPairs));
     } catch {
       // Storage full or unavailable — non-critical, skip persisting.
     }
@@ -410,6 +420,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
     mcpServerToolRestrictions,
     selectedVoice,
     streamingEnabled,
+    customHeaderPairs,
   ]);
 
   useEffect(() => {
@@ -921,6 +932,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
             mockTestFallbacks,
             mcpToolsets,
             streamingEnabled,
+            customHeaders,
           );
         } else if (endpointType === EndpointType.IMAGE) {
           // For image generation
@@ -932,6 +944,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
             selectedTags,
             signal,
             customProxyBaseUrl || undefined,
+            customHeaders,
           );
         } else if (endpointType === EndpointType.SPEECH) {
           // For audio speech
@@ -946,6 +959,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
             undefined, // responseFormat
             undefined, // speed
             customProxyBaseUrl || undefined,
+            customHeaders,
           );
         } else if (endpointType === EndpointType.IMAGE_EDITS) {
           // For image edits
@@ -959,6 +973,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
               selectedTags,
               signal,
               customProxyBaseUrl || undefined,
+              customHeaders,
             );
           }
         } else if (endpointType === EndpointType.RESPONSES) {
@@ -1004,6 +1019,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
             mcpToolsets,
             streamingEnabled,
             updateTotalLatency,
+            customHeaders,
           );
         } else if (endpointType === EndpointType.ANTHROPIC_MESSAGES) {
           const apiChatHistory = [
@@ -1033,6 +1049,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
             mcpServerToolRestrictions,
             mcpToolsets,
             streamingEnabled,
+            customHeaders,
           );
         } else if (endpointType === EndpointType.EMBEDDINGS) {
           await makeOpenAIEmbeddingsRequest(
@@ -1042,6 +1059,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
             effectiveApiKey,
             selectedTags,
             customProxyBaseUrl || undefined,
+            customHeaders,
           );
         } else if (endpointType === EndpointType.TRANSCRIPTION) {
           // For audio transcriptions
@@ -1058,6 +1076,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
               undefined, // responseFormat
               undefined, // temperature
               customProxyBaseUrl || undefined,
+              customHeaders,
             );
           }
         } else if (endpointType === EndpointType.INTERACTIONS) {
@@ -1069,6 +1088,8 @@ const ChatUI: React.FC<ChatUIProps> = ({
             selectedTags,
             signal,
             customProxyBaseUrl || undefined,
+            undefined,
+            customHeaders,
           );
         }
       }
@@ -1086,13 +1107,10 @@ const ChatUI: React.FC<ChatUIProps> = ({
           resolvedServerId = toolEntry?.server_id ?? rawSelected;
         }
         if (resolvedServerId && !resolvedServerId.startsWith("toolset:") && selectedMCPDirectTool) {
-          const result = await callMCPTool(
-            effectiveApiKey,
-            resolvedServerId,
-            selectedMCPDirectTool,
-            mcpToolArguments,
-            selectedGuardrails.length > 0 ? { guardrails: selectedGuardrails } : undefined,
-          );
+          const result = await callMCPTool(effectiveApiKey, resolvedServerId, selectedMCPDirectTool, mcpToolArguments, {
+            ...(selectedGuardrails.length > 0 ? { guardrails: selectedGuardrails } : {}),
+            customHeaders,
+          });
           const resultText =
             result?.content?.length > 0
               ? JSON.stringify(
@@ -1118,6 +1136,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
           updateA2AMetadata,
           customProxyBaseUrl || undefined,
           selectedGuardrails.length > 0 ? selectedGuardrails : undefined,
+          customHeaders,
         );
       }
     } catch (error) {
@@ -1485,6 +1504,18 @@ const ChatUI: React.FC<ChatUIProps> = ({
                   />
                 </div>
 
+                {endpointType !== EndpointType.REALTIME && (
+                  <div>
+                    <label className="mb-2 flex items-center text-sm font-medium text-foreground">
+                      <ListPlus className="mr-2 size-4" aria-hidden="true" /> Custom Headers
+                    </label>
+                    <KeyValueInput value={customHeaderPairs} onChange={setCustomHeaderPairs} />
+                    <p className="mt-2 text-xs text-muted-foreground">
+                      Sent with every playground request, e.g. provider-specific headers like anthropic-beta.
+                    </p>
+                  </div>
+                )}
+
                 <div>
                   <div className="mb-2 flex items-center gap-1 text-sm font-medium text-foreground">
                     <Wrench className="mr-1 size-4" aria-hidden="true" />
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx
index 0b97d9931c4..9440eb184bc 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx
@@ -3,6 +3,7 @@
 
 import { v4 as uuidv4 } from "uuid";
 import { getProxyBaseUrl, getGlobalLitellmHeaderName } from "@/components/networking";
+import type { CustomHeaders } from "@/components/llm_calls/request_headers";
 import { A2ATaskMetadata } from "@/components/chat_ui/types";
 
 interface A2AMessagePart {
@@ -116,6 +117,7 @@ export const makeA2ASendMessageRequest = async (
   onA2AMetadata?: (metadata: A2ATaskMetadata) => void,
   customBaseUrl?: string,
   guardrails?: string[],
+  customHeaders?: CustomHeaders,
 ): Promise<void> => {
   const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
   const url = proxyBaseUrl ? `${proxyBaseUrl}/a2a/${agentId}/message/send` : `/a2a/${agentId}/message/send`;
@@ -149,6 +151,7 @@ export const makeA2ASendMessageRequest = async (
       headers: {
         [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
         "Content-Type": "application/json",
+        ...customHeaders,
       },
       body: JSON.stringify(jsonRpcRequest),
       signal,
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx
index 9f030d8b2d4..bed6d9f4fd9 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.test.tsx
@@ -1,4 +1,5 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
+import Anthropic from "@anthropic-ai/sdk";
 import { makeAnthropicMessagesRequest } from "./anthropic_messages";
 import type { TokenUsage } from "@/components/chat_ui/ResponseMetrics";
 
@@ -122,4 +123,26 @@ describe("anthropic_messages non-streaming", () => {
     expect(mockMessagesCreate).not.toHaveBeenCalled();
     expect(mockMessagesStream.mock.calls[0][0]).toMatchObject({ stream: true });
   });
+
+  it("sends custom headers alongside the tags header on the Anthropic client", async () => {
+    mockMessagesCreate.mockResolvedValue({ content: [{ type: "text", text: "OK" }], usage: {} });
+
+    await makeAnthropicMessagesRequest(
+      [{ role: "user", content: "Hello" }],
+      vi.fn(),
+      "claude-haiku-4-5",
+      "test-token",
+      ["team-a"],
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      ...NON_STREAMING_ARGS,
+      { "anthropic-beta": "context-1m-2025-08-07" },
+    );
+
+    expect(vi.mocked(Anthropic).mock.calls[0][0]).toMatchObject({
+      defaultHeaders: { "x-litellm-tags": "team-a", "anthropic-beta": "context-1m-2025-08-07" },
+    });
+  });
 });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx
index 9dd2f675c44..4143d2641e8 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/anthropic_messages.tsx
@@ -2,6 +2,7 @@ import Anthropic from "@anthropic-ai/sdk";
 import { MessageType } from "@/components/chat_ui/types";
 import { TokenUsage } from "@/components/chat_ui/ResponseMetrics";
 import { buildMcpToolBlocks } from "@/components/llm_calls/mcp_tool_blocks";
+import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
 import { MCPServer, MCPToolset } from "@/components/mcp_tools/types";
 import { getProxyBaseUrl } from "@/components/networking";
 import { toast } from "@/lib/toast";
@@ -34,6 +35,7 @@ export async function makeAnthropicMessagesRequest(
   mcpServerToolRestrictions?: Record<string, string[]>,
   mcpToolsets?: MCPToolset[],
   streamingEnabled: boolean = true,
+  customHeaders?: CustomHeaders,
 ) {
   if (!accessToken) {
     throw new Error("Virtual Key is required");
@@ -46,11 +48,7 @@ export async function makeAnthropicMessagesRequest(
 
   const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
 
-  // Prepare headers with tags and trace ID
-  const headers: Record<string, string> = {};
-  if (tags && tags.length > 0) {
-    headers["x-litellm-tags"] = tags.join(",");
-  }
+  const headers = buildPlaygroundHeaders(tags, customHeaders);
 
   const client = new Anthropic({
     apiKey: accessToken,
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_speech.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_speech.tsx
index d9361bd0777..367bd9aa6a6 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_speech.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_speech.tsx
@@ -1,5 +1,6 @@
 import openai from "openai";
 import { getProxyBaseUrl } from "@/components/networking";
+import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
 import { toast } from "@/lib/toast";
 import type { OpenAIVoice } from "../components/chat_ui/chatConstants";
 
@@ -14,6 +15,7 @@ export async function makeOpenAIAudioSpeechRequest(
   responseFormat?: string,
   speed?: number,
   customBaseUrl?: string,
+  customHeaders?: CustomHeaders,
 ) {
   // base url should be the current base_url
   const isLocal = process.env.NODE_ENV === "development";
@@ -25,7 +27,7 @@ export async function makeOpenAIAudioSpeechRequest(
     apiKey: accessToken,
     baseURL: proxyBaseUrl,
     dangerouslyAllowBrowser: true,
-    defaultHeaders: tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : undefined,
+    defaultHeaders: buildPlaygroundHeaders(tags, customHeaders),
   });
 
   try {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_transcriptions.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_transcriptions.tsx
index 50d5fb19990..12b2d945f98 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_transcriptions.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/audio_transcriptions.tsx
@@ -1,5 +1,6 @@
 import openai from "openai";
 import { getProxyBaseUrl } from "@/components/networking";
+import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
 import { toast } from "@/lib/toast";
 
 export async function makeOpenAIAudioTranscriptionRequest(
@@ -14,6 +15,7 @@ export async function makeOpenAIAudioTranscriptionRequest(
   responseFormat?: string,
   temperature?: number,
   customBaseUrl?: string,
+  customHeaders?: CustomHeaders,
 ) {
   // base url should be the current base_url
   const isLocal = process.env.NODE_ENV === "development";
@@ -26,7 +28,7 @@ export async function makeOpenAIAudioTranscriptionRequest(
     apiKey: accessToken,
     baseURL: proxyBaseUrl,
     dangerouslyAllowBrowser: true,
-    defaultHeaders: tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : undefined,
+    defaultHeaders: buildPlaygroundHeaders(tags, customHeaders),
   });
 
   try {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx
index 19a79f3f9be..ecdb47af9a8 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx
@@ -76,4 +76,24 @@ describe("embeddings_api", () => {
       input: "Sample text",
     });
   });
+
+  it("sends custom headers on the fetch request, letting them override the tags header", async () => {
+    await makeOpenAIEmbeddingsRequest(
+      "Sample text",
+      mockUpdateEmbeddingsUI,
+      "text-embedding-3-small",
+      "abcdef",
+      ["team-a"],
+      undefined,
+      { "x-litellm-tags": "team-b", "x-request-source": "playground" },
+    );
+
+    expect(mockFetch.mock.calls[0][1]).toMatchObject({
+      headers: {
+        Authorization: "Bearer abcdef",
+        "x-litellm-tags": "team-b",
+        "x-request-source": "playground",
+      },
+    });
+  });
 });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx
index b43955b711d..ff94956694a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx
@@ -1,5 +1,6 @@
 import { toast } from "@/lib/toast";
 import { getProxyBaseUrl, getGlobalLitellmHeaderName } from "@/components/networking";
+import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
 
 export async function makeOpenAIEmbeddingsRequest(
   input: string,
@@ -8,6 +9,7 @@ export async function makeOpenAIEmbeddingsRequest(
   accessToken: string,
   tags?: string[],
   customBaseUrl?: string,
+  customHeaders?: CustomHeaders,
 ) {
   if (!accessToken) {
     throw new Error("Virtual Key is required");
@@ -20,11 +22,7 @@ export async function makeOpenAIEmbeddingsRequest(
   }
 
   const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
-  // Prepare headers with tags and trace ID
-  const headers: Record<string, string> = {};
-  if (tags && tags.length > 0) {
-    headers["x-litellm-tags"] = tags.join(",");
-  }
+  const headers = buildPlaygroundHeaders(tags, customHeaders);
 
   try {
     const normalizedBaseUrl = proxyBaseUrl.endsWith("/") ? proxyBaseUrl.slice(0, -1) : proxyBaseUrl;
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_edits.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_edits.tsx
index f66be10897d..b956cd1d7de 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_edits.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_edits.tsx
@@ -1,5 +1,6 @@
 import openai from "openai";
 import { getProxyBaseUrl } from "@/components/networking";
+import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
 import { toast } from "@/lib/toast";
 
 export async function makeOpenAIImageEditsRequest(
@@ -11,6 +12,7 @@ export async function makeOpenAIImageEditsRequest(
   tags?: string[],
   signal?: AbortSignal,
   customBaseUrl?: string,
+  customHeaders?: CustomHeaders,
 ) {
   // base url should be the current base_url
   const isLocal = process.env.NODE_ENV === "development";
@@ -23,7 +25,7 @@ export async function makeOpenAIImageEditsRequest(
     apiKey: accessToken,
     baseURL: proxyBaseUrl,
     dangerouslyAllowBrowser: true,
-    defaultHeaders: tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : undefined,
+    defaultHeaders: buildPlaygroundHeaders(tags, customHeaders),
   });
 
   try {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_generation.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_generation.tsx
index 7d3fb098257..1bbda4234b0 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_generation.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/image_generation.tsx
@@ -1,5 +1,6 @@
 import openai from "openai";
 import { getProxyBaseUrl } from "@/components/networking";
+import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
 import { toast } from "@/lib/toast";
 
 export async function makeOpenAIImageGenerationRequest(
@@ -10,6 +11,7 @@ export async function makeOpenAIImageGenerationRequest(
   tags?: string[],
   signal?: AbortSignal,
   customBaseUrl?: string,
+  customHeaders?: CustomHeaders,
 ) {
   // base url should be the current base_url
   const isLocal = process.env.NODE_ENV === "development";
@@ -21,7 +23,7 @@ export async function makeOpenAIImageGenerationRequest(
     apiKey: accessToken,
     baseURL: proxyBaseUrl,
     dangerouslyAllowBrowser: true,
-    defaultHeaders: tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : undefined,
+    defaultHeaders: buildPlaygroundHeaders(tags, customHeaders),
   });
 
   try {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx
index f1f925ca4e3..5897aafb9ae 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx
@@ -1,5 +1,6 @@
 import { toast } from "@/lib/toast";
 import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking";
+import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
 
 export async function makeInteractionsRequest(
   input: string,
@@ -10,6 +11,7 @@ export async function makeInteractionsRequest(
   signal?: AbortSignal,
   customBaseUrl?: string,
   previousInteractionId?: string,
+  customHeaders?: CustomHeaders,
 ): Promise<void> {
   if (!accessToken) {
     throw new Error("Virtual Key is required");
@@ -27,10 +29,8 @@ export async function makeInteractionsRequest(
   const headers: Record<string, string> = {
     "Content-Type": "application/json",
     [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
+    ...buildPlaygroundHeaders(tags, customHeaders),
   };
-  if (tags && tags.length > 0) {
-    headers["x-litellm-tags"] = tags.join(",");
-  }
 
   const body: Record<string, unknown> = {
     model: selectedModel,
diff --git a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx
index 0113f7b2832..65f5ade7a5c 100644
--- a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx
+++ b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.test.tsx
@@ -48,6 +48,28 @@ describe("CodeSnippets", () => {
     expect(code).toContain("print(response.data[0].embedding)");
   });
 
+  describe("custom headers", () => {
+    const customHeaders = { "anthropic-beta": "context-1m-2025-08-07", "x-request-source": "playground" };
+
+    it("passes configured headers as default_headers on the OpenAI client", () => {
+      const code = generateCodeSnippet({ ...baseParams, endpointType: EndpointType.CHAT, customHeaders });
+      expect(code).toContain('base_url="http://localhost:4000",\n\tdefault_headers={');
+      expect(code).toContain('"anthropic-beta": "context-1m-2025-08-07"');
+      expect(code).toContain('"x-request-source": "playground"');
+    });
+
+    it("passes configured headers as default_headers on the Azure client", () => {
+      const code = generateCodeSnippet({ ...baseParams, selectedSdk: "azure", customHeaders });
+      expect(code).toContain('api_version="2024-02-01",\n\tdefault_headers={');
+      expect(code).toContain('"anthropic-beta": "context-1m-2025-08-07"');
+    });
+
+    it("omits default_headers when no custom headers are configured", () => {
+      expect(generateCodeSnippet(baseParams)).not.toContain("default_headers");
+      expect(generateCodeSnippet({ ...baseParams, customHeaders: {} })).not.toContain("default_headers");
+    });
+  });
+
   describe("base URL selection", () => {
     it("should use LITELLM_UI_API_DOC_BASE_URL when provided", () => {
       const customBaseUrl = "https://custom-doc.example.com";
diff --git a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx
index f458e563b4d..9150ba0e632 100644
--- a/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx
+++ b/ui/litellm-dashboard/src/components/chat_ui/CodeSnippets.tsx
@@ -1,6 +1,7 @@
 import { MessageType } from "./types";
 import { EndpointType } from "./mode_endpoint_mapping";
 import { MCPServer } from "@/components/mcp_tools/types";
+import type { CustomHeaders } from "@/components/llm_calls/request_headers";
 
 interface CodeGenMetadata {
   tags?: string[];
@@ -30,6 +31,7 @@ interface GenerateCodeParams {
     PROXY_BASE_URL?: string;
     LITELLM_UI_API_DOC_BASE_URL?: string | null;
   };
+  customHeaders?: CustomHeaders;
 }
 
 export const generateCodeSnippet = (params: GenerateCodeParams): string => {
@@ -48,6 +50,7 @@ export const generateCodeSnippet = (params: GenerateCodeParams): string => {
     selectedModel,
     selectedSdk,
     proxySettings,
+    customHeaders,
   } = params;
   const effectiveApiKey = apiKeySource === "session" ? accessToken : apiKey;
 
@@ -76,6 +79,11 @@ export const generateCodeSnippet = (params: GenerateCodeParams): string => {
 
   const modelNameForCode = selectedModel || "your-model-name";
 
+  const defaultHeadersCode =
+    customHeaders && Object.keys(customHeaders).length > 0
+      ? `,\n\tdefault_headers=${JSON.stringify(customHeaders, null, 2).replace(/\n/g, "\n\t")}`
+      : "";
+
   const clientInitialization =
     selectedSdk === "azure"
       ? `import openai
@@ -83,13 +91,13 @@ export const generateCodeSnippet = (params: GenerateCodeParams): string => {
 client = openai.AzureOpenAI(
 	api_key="${effectiveApiKey || "YOUR_LITELLM_API_KEY"}",
 	azure_endpoint="${apiBase}",
-	api_version="2024-02-01"
+	api_version="2024-02-01"${defaultHeadersCode}
 )`
       : `import openai
 
 client = openai.OpenAI(
 	api_key="${effectiveApiKey || "YOUR_LITELLM_API_KEY"}",
-	base_url="${apiBase}"
+	base_url="${apiBase}"${defaultHeadersCode}
 )`;
 
   let endpointSpecificCode;
diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
index ec477441586..f10502df77b 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
@@ -1,4 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import openai from "openai";
 import { makeOpenAIChatCompletionRequest } from "./chat_completion";
 import type { TokenUsage } from "../chat_ui/ResponseMetrics";
 
@@ -615,3 +616,47 @@ describe("chat_completion response cache", () => {
     expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true }));
   });
 });
+
+describe("chat_completion custom headers", () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("sends custom headers alongside the tags header on the OpenAI client", async () => {
+    mockCreate.mockReturnValueOnce(nonStreamingResponse({ choices: [{ message: { content: "Hi" } }] }));
+
+    await makeOpenAIChatCompletionRequest(
+      [{ role: "user", content: "Hello" }],
+      vi.fn(),
+      "gpt-4",
+      "test-token",
+      ["team-a"],
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      false,
+      { "anthropic-beta": "context-1m-2025-08-07", "x-litellm-tags": "overridden" },
+    );
+
+    expect(vi.mocked(openai.OpenAI).mock.calls[0][0]).toMatchObject({
+      defaultHeaders: { "anthropic-beta": "context-1m-2025-08-07", "x-litellm-tags": "overridden" },
+    });
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
index ffa2877fbd9..fe5b6fb6e39 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
@@ -6,6 +6,7 @@ import { getProxyBaseUrl } from "@/components/networking";
 import { MCPServer, MCPToolset, type MCPEvent } from "@/components/mcp_tools/types";
 import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
 import { parseUsageCost } from "./usage_cost";
+import { buildPlaygroundHeaders, type CustomHeaders } from "./request_headers";
 
 const completionAsSingleChunk = (completion: ChatCompletion): ChatCompletionChunk =>
   ({
@@ -50,6 +51,7 @@ export async function makeOpenAIChatCompletionRequest(
   mockTestFallbacks?: boolean,
   mcpToolsets?: MCPToolset[],
   streamingEnabled: boolean = true,
+  customHeaders?: CustomHeaders,
 ) {
   // base url should be the current base_url
   const isLocal = process.env.NODE_ENV === "development";
@@ -57,11 +59,7 @@ export async function makeOpenAIChatCompletionRequest(
     console.log = function () {};
   }
   const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
-  // Prepare headers with tags and trace ID
-  const headers: Record<string, string> = {};
-  if (tags && tags.length > 0) {
-    headers["x-litellm-tags"] = tags.join(",");
-  }
+  const headers = buildPlaygroundHeaders(tags, customHeaders);
 
   const client = new openai.OpenAI({
     apiKey: accessToken,
diff --git a/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts b/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts
new file mode 100644
index 00000000000..121babbab26
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts
@@ -0,0 +1,44 @@
+import { describe, expect, it } from "vitest";
+import { buildPlaygroundHeaders, customHeadersFromPairs, parseStoredHeaderPairs } from "./request_headers";
+
+describe("customHeadersFromPairs", () => {
+  it("trims header names and drops rows without a name", () => {
+    expect(
+      customHeadersFromPairs([
+        [" anthropic-beta ", "context-1m-2025-08-07"],
+        ["", "orphan value"],
+        ["   ", "whitespace name"],
+        ["x-empty", ""],
+      ]),
+    ).toEqual({ "anthropic-beta": "context-1m-2025-08-07", "x-empty": "" });
+  });
+});
+
+describe("parseStoredHeaderPairs", () => {
+  it("round-trips pairs persisted as JSON", () => {
+    const pairs = [["anthropic-beta", "context-1m-2025-08-07"]] as const;
+    expect(parseStoredHeaderPairs(JSON.stringify(pairs))).toEqual(pairs);
+  });
+
+  it("returns no pairs for missing, malformed, or wrongly shaped storage", () => {
+    expect(parseStoredHeaderPairs(null)).toEqual([]);
+    expect(parseStoredHeaderPairs("not json")).toEqual([]);
+    expect(parseStoredHeaderPairs(JSON.stringify({ "anthropic-beta": "x" }))).toEqual([]);
+    expect(parseStoredHeaderPairs(JSON.stringify([["ok", "pair"], ["one"], [1, 2], "str"]))).toEqual([["ok", "pair"]]);
+  });
+});
+
+describe("buildPlaygroundHeaders", () => {
+  it("joins tags into x-litellm-tags and lets custom headers override it", () => {
+    expect(buildPlaygroundHeaders(["a", "b"], { "x-custom": "1" })).toEqual({
+      "x-litellm-tags": "a,b",
+      "x-custom": "1",
+    });
+    expect(buildPlaygroundHeaders(["a"], { "x-litellm-tags": "b" })).toEqual({ "x-litellm-tags": "b" });
+  });
+
+  it("omits x-litellm-tags when there are no tags", () => {
+    expect(buildPlaygroundHeaders([], { "x-custom": "1" })).toEqual({ "x-custom": "1" });
+    expect(buildPlaygroundHeaders(undefined, undefined)).toEqual({});
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts b/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts
new file mode 100644
index 00000000000..13e9631e453
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts
@@ -0,0 +1,27 @@
+import type { KeyValuePair } from "@/components/key_value_input";
+
+export type CustomHeaders = Readonly<Record<string, string>>;
+
+export const customHeadersFromPairs = (pairs: readonly KeyValuePair[]): CustomHeaders =>
+  Object.fromEntries(pairs.map(([name, value]) => [name.trim(), value]).filter(([name]) => name !== ""));
+
+const isHeaderPair = (entry: unknown): entry is KeyValuePair =>
+  Array.isArray(entry) && entry.length === 2 && entry.every((part) => typeof part === "string");
+
+export const parseStoredHeaderPairs = (raw: string | null): readonly KeyValuePair[] => {
+  if (!raw) return [];
+  try {
+    const parsed: unknown = JSON.parse(raw);
+    return Array.isArray(parsed) ? parsed.filter(isHeaderPair) : [];
+  } catch {
+    return [];
+  }
+};
+
+export const buildPlaygroundHeaders = (
+  tags?: readonly string[],
+  customHeaders?: CustomHeaders,
+): Record<string, string> => ({
+  ...(tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : {}),
+  ...customHeaders,
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
index 290c8b0b619..a2260d25ca6 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
@@ -1,4 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import openai from "openai";
 import { makeOpenAIResponsesRequest } from "./responses_api";
 import { MessageType } from "../chat_ui/types";
 import type { TokenUsage } from "../chat_ui/ResponseMetrics";
@@ -611,3 +612,46 @@ describe("responses_api response cache", () => {
     expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true }), "");
   });
 });
+
+describe("responses_api custom headers", () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("sends custom headers alongside the tags header on the OpenAI client", async () => {
+    mockResponsesCreate.mockReturnValueOnce(nonStreamingResponse({ id: "resp_1", output: [] }));
+
+    await makeOpenAIResponsesRequest(
+      [{ role: "user", content: "Hello" }],
+      vi.fn(),
+      "gpt-4",
+      "test-token",
+      ["team-a"],
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      false,
+      undefined,
+      { "anthropic-beta": "context-1m-2025-08-07" },
+    );
+
+    expect(vi.mocked(openai.OpenAI).mock.calls[0][0]).toMatchObject({
+      defaultHeaders: { "x-litellm-tags": "team-a", "anthropic-beta": "context-1m-2025-08-07" },
+    });
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
index ce54c7c6b40..ec196a8d9a2 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
@@ -5,6 +5,7 @@ import { getProxyBaseUrl } from "@/components/networking";
 import { toast } from "@/lib/toast";
 import { extractPromptCacheTokens } from "@/utils/promptCacheUsage";
 import { parseUsageCost } from "./usage_cost";
+import { buildPlaygroundHeaders, type CustomHeaders } from "./request_headers";
 import type { MCPEvent } from "@/components/mcp_tools/types";
 import { MCPServer, MCPToolset } from "@/components/mcp_tools/types";
 import {
@@ -85,6 +86,7 @@ export async function makeOpenAIResponsesRequest(
   mcpToolsets?: MCPToolset[],
   streamingEnabled: boolean = true,
   onTotalLatency?: (latency: number) => void,
+  customHeaders?: CustomHeaders,
 ) {
   if (!accessToken) {
     throw new Error("Virtual Key is required");
@@ -101,11 +103,7 @@ export async function makeOpenAIResponsesRequest(
   }
 
   const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
-  // Prepare headers with tags and trace ID
-  const headers: Record<string, string> = {};
-  if (tags && tags.length > 0) {
-    headers["x-litellm-tags"] = tags.join(",");
-  }
+  const headers = buildPlaygroundHeaders(tags, customHeaders);
 
   const client = new openai.OpenAI({
     apiKey: accessToken,

From 9261a72eb7f9386caec4e00be0ac488e88fd8503 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:14:44 +0000
Subject: [PATCH 37/67] test(passthrough): cover Vertex router resolution
 without a matching deployment

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../test_vertex_passthrough_load_balancing.py | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
index dde47004adc..29a635e9b27 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
@@ -8,6 +8,7 @@ from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     VertexAIPassThroughHandler,
     _base_vertex_proxy_route,
+    _resolve_vertex_model_from_router,
     _upstream_headers_for_vertex_route,
 )
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
@@ -835,3 +836,43 @@ async def test_vertex_passthrough_attributes_the_call_to_the_resolved_deployment
         )
 
     assert logging_kwargs["litellm_params"]["metadata"]["model_info"]["id"] == "vertex-gemini-38-flash-dep"
+
+
+def _router_without_deployment() -> MagicMock:
+    router = MagicMock()
+    router.get_available_deployment_for_pass_through.return_value = None
+    return router
+
+
+def _router_raising_on_lookup() -> MagicMock:
+    router = MagicMock()
+    router.get_available_deployment_for_pass_through.side_effect = ValueError("no healthy deployment")
+    return router
+
+
+@pytest.mark.parametrize(
+    "llm_router",
+    [None, _router_without_deployment(), _router_raising_on_lookup()],
+    ids=["no-router", "no-matching-deployment", "lookup-raises"],
+)
+def test_vertex_passthrough_without_a_resolved_deployment_keeps_the_url_and_reports_no_model_info(
+    llm_router: MagicMock | None,
+):
+    """A Vertex passthrough call that no router deployment serves must keep the URL-derived values and carry no
+    deployment model_info, so logging cannot attribute it to a deployment that never handled it."""
+    resolved = _resolve_vertex_model_from_router(
+        model_id="gemini-3.8-flash",
+        llm_router=llm_router,
+        encoded_endpoint="/v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent",
+        endpoint="v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent",
+        vertex_project="url-project",
+        vertex_location="url-location",
+    )
+
+    assert resolved == (
+        "/v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent",
+        "v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent",
+        "url-project",
+        "url-location",
+        None,
+    )

From 754e87fa670422edaba3733f2dbb533c4644da1d Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:24:13 +0000
Subject: [PATCH 38/67] fix(ui): keep gateway auth and content-type headers
 ahead of playground custom headers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../playground/llm_calls/a2a_send_message.tsx  |  7 +++----
 .../llm_calls/embeddings_api.test.tsx          | 18 ++++++++++++++++++
 .../playground/llm_calls/embeddings_api.tsx    | 17 ++++++++++-------
 .../playground/llm_calls/interactions_api.tsx  | 11 +++++++----
 .../llm_calls/request_headers.test.ts          | 18 +++++++++++++++++-
 .../components/llm_calls/request_headers.ts    | 11 +++++++++++
 6 files changed, 66 insertions(+), 16 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx
index 9440eb184bc..db14290f42e 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/a2a_send_message.tsx
@@ -3,7 +3,7 @@
 
 import { v4 as uuidv4 } from "uuid";
 import { getProxyBaseUrl, getGlobalLitellmHeaderName } from "@/components/networking";
-import type { CustomHeaders } from "@/components/llm_calls/request_headers";
+import { type CustomHeaders, withRequiredHeaders } from "@/components/llm_calls/request_headers";
 import { A2ATaskMetadata } from "@/components/chat_ui/types";
 
 interface A2AMessagePart {
@@ -148,11 +148,10 @@ export const makeA2ASendMessageRequest = async (
   try {
     const response = await fetch(url, {
       method: "POST",
-      headers: {
+      headers: withRequiredHeaders(customHeaders ?? {}, {
         [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
         "Content-Type": "application/json",
-        ...customHeaders,
-      },
+      }),
       body: JSON.stringify(jsonRpcRequest),
       signal,
     });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx
index ecdb47af9a8..ef5c9f5025d 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.test.tsx
@@ -96,4 +96,22 @@ describe("embeddings_api", () => {
       },
     });
   });
+
+  it("does not let custom headers replace the gateway auth or content-type headers", async () => {
+    await makeOpenAIEmbeddingsRequest(
+      "Sample text",
+      mockUpdateEmbeddingsUI,
+      "text-embedding-3-small",
+      "abcdef",
+      undefined,
+      undefined,
+      { authorization: "Bearer stolen", "Content-Type": "text/plain", "x-request-source": "playground" },
+    );
+
+    expect(mockFetch.mock.calls[0][1].headers).toEqual({
+      Authorization: "Bearer abcdef",
+      "Content-Type": "application/json",
+      "x-request-source": "playground",
+    });
+  });
 });
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx
index ff94956694a..ef8a650f1d2 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/embeddings_api.tsx
@@ -1,6 +1,10 @@
 import { toast } from "@/lib/toast";
 import { getProxyBaseUrl, getGlobalLitellmHeaderName } from "@/components/networking";
-import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
+import {
+  buildPlaygroundHeaders,
+  type CustomHeaders,
+  withRequiredHeaders,
+} from "@/components/llm_calls/request_headers";
 
 export async function makeOpenAIEmbeddingsRequest(
   input: string,
@@ -22,7 +26,10 @@ export async function makeOpenAIEmbeddingsRequest(
   }
 
   const proxyBaseUrl = customBaseUrl || getProxyBaseUrl();
-  const headers = buildPlaygroundHeaders(tags, customHeaders);
+  const headers = withRequiredHeaders(buildPlaygroundHeaders(tags, customHeaders), {
+    "Content-Type": "application/json",
+    [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
+  });
 
   try {
     const normalizedBaseUrl = proxyBaseUrl.endsWith("/") ? proxyBaseUrl.slice(0, -1) : proxyBaseUrl;
@@ -30,11 +37,7 @@ export async function makeOpenAIEmbeddingsRequest(
 
     const response = await fetch(requestUrl, {
       method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
-        ...headers,
-      },
+      headers,
       body: JSON.stringify({
         model: selectedModel,
         input,
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx
index 5897aafb9ae..ab1bb7c1707 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/interactions_api.tsx
@@ -1,6 +1,10 @@
 import { toast } from "@/lib/toast";
 import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking";
-import { buildPlaygroundHeaders, type CustomHeaders } from "@/components/llm_calls/request_headers";
+import {
+  buildPlaygroundHeaders,
+  type CustomHeaders,
+  withRequiredHeaders,
+} from "@/components/llm_calls/request_headers";
 
 export async function makeInteractionsRequest(
   input: string,
@@ -26,11 +30,10 @@ export async function makeInteractionsRequest(
   const normalizedBaseUrl = proxyBaseUrl.endsWith("/") ? proxyBaseUrl.slice(0, -1) : proxyBaseUrl;
   const requestUrl = `${normalizedBaseUrl}/v1beta/interactions`;
 
-  const headers: Record<string, string> = {
+  const headers: Record<string, string> = withRequiredHeaders(buildPlaygroundHeaders(tags, customHeaders), {
     "Content-Type": "application/json",
     [getGlobalLitellmHeaderName()]: `Bearer ${accessToken}`,
-    ...buildPlaygroundHeaders(tags, customHeaders),
-  };
+  });
 
   const body: Record<string, unknown> = {
     model: selectedModel,
diff --git a/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts b/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts
index 121babbab26..cacedea3c87 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts
+++ b/ui/litellm-dashboard/src/components/llm_calls/request_headers.test.ts
@@ -1,5 +1,10 @@
 import { describe, expect, it } from "vitest";
-import { buildPlaygroundHeaders, customHeadersFromPairs, parseStoredHeaderPairs } from "./request_headers";
+import {
+  buildPlaygroundHeaders,
+  customHeadersFromPairs,
+  parseStoredHeaderPairs,
+  withRequiredHeaders,
+} from "./request_headers";
 
 describe("customHeadersFromPairs", () => {
   it("trims header names and drops rows without a name", () => {
@@ -42,3 +47,14 @@ describe("buildPlaygroundHeaders", () => {
     expect(buildPlaygroundHeaders(undefined, undefined)).toEqual({});
   });
 });
+
+describe("withRequiredHeaders", () => {
+  it("keeps required headers regardless of custom header name casing", () => {
+    expect(
+      withRequiredHeaders(
+        { authorization: "Bearer stolen", "content-type": "text/plain", "x-custom": "1" },
+        { Authorization: "Bearer real", "Content-Type": "application/json" },
+      ),
+    ).toEqual({ Authorization: "Bearer real", "Content-Type": "application/json", "x-custom": "1" });
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts b/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts
index 13e9631e453..8c0c0056fca 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts
+++ b/ui/litellm-dashboard/src/components/llm_calls/request_headers.ts
@@ -25,3 +25,14 @@ export const buildPlaygroundHeaders = (
   ...(tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : {}),
   ...customHeaders,
 });
+
+export const withRequiredHeaders = (
+  headers: Readonly<Record<string, string>>,
+  required: Readonly<Record<string, string>>,
+): Record<string, string> => {
+  const reserved = new Set(Object.keys(required).map((name) => name.toLowerCase()));
+  return {
+    ...Object.fromEntries(Object.entries(headers).filter(([name]) => !reserved.has(name.toLowerCase()))),
+    ...required,
+  };
+};

From 168607cc3f18c7078fa5c3ea2604dc81a2f7c2c2 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:26:49 +0000
Subject: [PATCH 39/67] fix(proxy): register /nvidia_nim/ with the lazy
 passthrough loader

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/_lazy_features.py           |   1 +
 litellm/proxy/_lazy_openapi_snapshot.json | 222 ++++++++++++++++++++++
 2 files changed, 223 insertions(+)

diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py
index dd1180b30ad..faf95397fa5 100644
--- a/litellm/proxy/_lazy_features.py
+++ b/litellm/proxy/_lazy_features.py
@@ -205,6 +205,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
             "/gigachat/",
             "/milvus/",
             "/mistral/",
+            "/nvidia_nim/",
             "/openai/",
             "/openai_passthrough/",
             "/vertex-ai/",
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index f3b579d22c7..ad08598e760 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -18912,6 +18912,228 @@
           ]
         }
       },
+      "/nvidia_nim/{endpoint}": {
+        "delete": {
+          "description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
+          "operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__delete",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "endpoint",
+              "required": true,
+              "schema": {
+                "title": "Endpoint",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Nvidia Nim Proxy Route",
+          "tags": [
+            "llm_passthrough"
+          ]
+        },
+        "get": {
+          "description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
+          "operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__get",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "endpoint",
+              "required": true,
+              "schema": {
+                "title": "Endpoint",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Nvidia Nim Proxy Route",
+          "tags": [
+            "llm_passthrough"
+          ]
+        },
+        "patch": {
+          "description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
+          "operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__patch",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "endpoint",
+              "required": true,
+              "schema": {
+                "title": "Endpoint",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Nvidia Nim Proxy Route",
+          "tags": [
+            "llm_passthrough"
+          ]
+        },
+        "post": {
+          "description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
+          "operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "endpoint",
+              "required": true,
+              "schema": {
+                "title": "Endpoint",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Nvidia Nim Proxy Route",
+          "tags": [
+            "llm_passthrough"
+          ]
+        },
+        "put": {
+          "description": "Relay a native NVIDIA NIM request through a LiteLLM model group.\n\n`{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's\n`api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through\nvirtual key auth, model access checks, and spend logging.",
+          "operationId": "nvidia_nim_proxy_route_nvidia_nim__endpoint__put",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "endpoint",
+              "required": true,
+              "schema": {
+                "title": "Endpoint",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Nvidia Nim Proxy Route",
+          "tags": [
+            "llm_passthrough"
+          ]
+        }
+      },
       "/openai/deployments/{model}/chat/completions": {
         "post": {
           "description": "Follows the exact same API spec as `OpenAI's Chat API https://platform.openai.com/docs/api-reference/chat`\n\n```bash\ncurl -X POST http://localhost:4000/v1/chat/completions \n-H \"Content-Type: application/json\" \n-H \"Authorization: Bearer sk-1234\" \n-d '{\n    \"model\": \"gpt-4o\",\n    \"messages\": [\n        {\n            \"role\": \"user\",\n            \"content\": \"Hello!\"\n        }\n    ]\n}'\n```",

From 1b6e76af16efc781e10899c7dd433e2447d59dc0 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:27:24 +0000
Subject: [PATCH 40/67] fix(proxy): resolve LITELLM_LOG for uvicorn at startup
 and restore logger state in tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm-proxy-extras/tests/test_logging.py | 37 ++++++++++++----------
 litellm/_logging.py                        |  6 +++-
 litellm/proxy/proxy_cli.py                 |  6 ++--
 tests/test_litellm/proxy/test_proxy_cli.py | 14 +++-----
 4 files changed, 33 insertions(+), 30 deletions(-)

diff --git a/litellm-proxy-extras/tests/test_logging.py b/litellm-proxy-extras/tests/test_logging.py
index b54d4fb72c9..fb209bc3925 100644
--- a/litellm-proxy-extras/tests/test_logging.py
+++ b/litellm-proxy-extras/tests/test_logging.py
@@ -1,34 +1,37 @@
 import importlib
 import logging
+from collections.abc import Iterator
 
 import pytest
 
 import litellm_proxy_extras._logging as extras_logging
 
 
-def test_litellm_log_error_silences_extras_info_lines(monkeypatch):
-    saved_handlers = logging.getLogger("litellm_proxy_extras").handlers[:]
-    monkeypatch.setenv("LITELLM_LOG", "ERROR")
-    logging.getLogger("litellm_proxy_extras").handlers[:] = []
+@pytest.fixture
+def fresh_extras_logger() -> Iterator[logging.Logger]:
+    logger = logging.getLogger("litellm_proxy_extras")
+    saved_handlers = logger.handlers[:]
+    saved_level = logger.level
+    logger.handlers[:] = []
     try:
-        reloaded = importlib.reload(extras_logging).logger
-        assert reloaded.isEnabledFor(logging.INFO) is False
-        assert reloaded.isEnabledFor(logging.ERROR) is True
+        yield logger
     finally:
-        logging.getLogger("litellm_proxy_extras").handlers[:] = saved_handlers
-        logging.getLogger("litellm_proxy_extras").setLevel(logging.INFO)
+        logger.handlers[:] = saved_handlers
+        logger.setLevel(saved_level)
+
+
+def test_litellm_log_error_silences_extras_info_lines(monkeypatch, fresh_extras_logger):
+    monkeypatch.setenv("LITELLM_LOG", "ERROR")
+    reloaded = importlib.reload(extras_logging).logger
+    assert reloaded is fresh_extras_logger
+    assert reloaded.isEnabledFor(logging.INFO) is False
+    assert reloaded.isEnabledFor(logging.ERROR) is True
 
 
 @pytest.mark.parametrize("litellm_log", [None, "info", "DEBUG"])
-def test_unset_or_verbose_litellm_log_keeps_extras_info_lines(monkeypatch, litellm_log):
-    saved_handlers = logging.getLogger("litellm_proxy_extras").handlers[:]
+def test_unset_or_verbose_litellm_log_keeps_extras_info_lines(monkeypatch, fresh_extras_logger, litellm_log):
     if litellm_log is None:
         monkeypatch.delenv("LITELLM_LOG", raising=False)
     else:
         monkeypatch.setenv("LITELLM_LOG", litellm_log)
-    logging.getLogger("litellm_proxy_extras").handlers[:] = []
-    try:
-        assert importlib.reload(extras_logging).logger.isEnabledFor(logging.INFO) is True
-    finally:
-        logging.getLogger("litellm_proxy_extras").handlers[:] = saved_handlers
-        logging.getLogger("litellm_proxy_extras").setLevel(logging.INFO)
+    assert importlib.reload(extras_logging).logger.isEnabledFor(logging.INFO) is True
diff --git a/litellm/_logging.py b/litellm/_logging.py
index 893133d62fe..873a6619a81 100644
--- a/litellm/_logging.py
+++ b/litellm/_logging.py
@@ -401,10 +401,14 @@ def _parse_json_logs_env(value: str | None) -> bool:
     return (value or "").lower() == "true"
 
 
+def resolve_log_level(log_level: str) -> int:
+    return getattr(logging, log_level.upper())
+
+
 json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
-numeric_level: Final[int] = getattr(logging, log_level.upper())
+numeric_level: Final[int] = resolve_log_level(log_level)
 handler: Final = LevelRoutingStreamHandler()
 handler.setLevel(numeric_level)
 handler.addFilter(_secret_filter)
diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py
index 6a23b422717..b25c77f6828 100644
--- a/litellm/proxy/proxy_cli.py
+++ b/litellm/proxy/proxy_cli.py
@@ -261,7 +261,7 @@ class ProxyInitializationHelpers:
         import uvicorn
 
         import litellm
-        from litellm._logging import _get_uvicorn_json_log_config, numeric_level
+        from litellm._logging import _get_uvicorn_json_log_config, resolve_log_level
 
         uvicorn_args: Final = {
             "app": "litellm.proxy.proxy_server:app",
@@ -275,8 +275,8 @@ class ProxyInitializationHelpers:
         elif litellm.json_logs:
             # Use JSON log config for uvicorn to ensure all logs (including exceptions) are JSON
             uvicorn_args["log_config"] = _get_uvicorn_json_log_config()
-        elif os.environ.get("LITELLM_LOG"):
-            uvicorn_args["log_level"] = numeric_level
+        elif litellm_log := os.environ.get("LITELLM_LOG"):
+            uvicorn_args["log_level"] = resolve_log_level(litellm_log)
         if keepalive_timeout is not None:
             uvicorn_args["timeout_keep_alive"] = keepalive_timeout
         if timeout_worker_healthcheck is not None:
diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py
index 8f3c21e10c2..712c526b244 100644
--- a/tests/test_litellm/proxy/test_proxy_cli.py
+++ b/tests/test_litellm/proxy/test_proxy_cli.py
@@ -143,24 +143,20 @@ class TestProxyInitializationHelpers:
     def _uvicorn_access_info_enabled(args: dict) -> bool:
         import logging
 
-        names = ("uvicorn", "uvicorn.error", "uvicorn.access", "uvicorn.asgi")
-        saved = tuple((logging.getLogger(n), logging.getLogger(n).handlers[:], logging.getLogger(n).level) for n in names)
+        loggers = tuple(logging.getLogger(n) for n in ("uvicorn", "uvicorn.error", "uvicorn.access", "uvicorn.asgi"))
+        saved = tuple((lg, lg.handlers[:], lg.level, lg.propagate) for lg in loggers)
         try:
             uvicorn.Config(**args).configure_logging()
             return logging.getLogger("uvicorn.access").isEnabledFor(logging.INFO)
         finally:
-            for lg, handlers, level in saved:
+            for lg, handlers, level, propagate in saved:
                 lg.handlers[:] = handlers
                 lg.setLevel(level)
+                lg.propagate = propagate
 
     def test_litellm_log_error_silences_uvicorn_info_lines(self, monkeypatch):
-        import logging
-
         monkeypatch.setenv("LITELLM_LOG", "ERROR")
-        with patch(  # test-quality-ok: numeric_level is resolved from LITELLM_LOG once at import; no other way to set it
-            "litellm._logging.numeric_level", logging.ERROR
-        ):
-            args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
+        args = ProxyInitializationHelpers._get_default_unvicorn_init_args("localhost", 8000)
 
         assert "log_config" not in args
         assert self._uvicorn_access_info_enabled(args) is False

From 5d6e367d562ef3e32f6c933867b575ad0845599f Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:32:11 +0000
Subject: [PATCH 41/67] fix(proxy): keep alias rewrite off pass-through bodies
 and auth-merged params

Skip router_settings.model_group_alias resolution on registered pass-through
routes, rebuild the rewritten body from the cached client payload instead of the
auth-enriched request_data, and centralize the resolved-scope sentinel in
litellm/constants.py

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/constants.py                          |  1 +
 litellm/proxy/auth/user_api_key_auth.py       | 28 +++++---------
 .../proxy/common_utils/http_parsing_utils.py  | 18 +++++++++
 .../proxy/auth/test_user_api_key_auth.py      | 38 +++++++++++++++++++
 4 files changed, 66 insertions(+), 19 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index ca2be8af5fe..d84effde139 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -318,6 +318,7 @@ BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_real
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
 BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
 CLIENT_REQUESTED_MODEL_SCOPE_KEY: Final = "litellm.client_requested_model"
+MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolved"
 REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
 REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
 
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index a02661db9ec..ac0f3696680 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -32,6 +32,7 @@ from litellm.constants import (
     INVALID_VIRTUAL_KEY_ERROR_MESSAGE,
     LITELLM_PROXY_BUDGET_NAME,
     LITELLM_PROXY_MASTER_KEY_ALIAS,
+    MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY,
 )
 from litellm.integrations.otel.model.config import is_otel_v2_enabled
 from litellm.integrations.otel.runtime import phase_span, seed_request_identity
@@ -104,6 +105,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
     _safe_set_request_parsed_body,
     populate_request_with_path_params,
     read_raw_json_body,
+    rewrite_request_model,
 )
 from litellm.proxy.common_utils.model_listing_utils import claude_code_requested_group
 from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
@@ -237,22 +239,7 @@ async def _normalize_claude_model(
         request.scope[_CLAUDE_MODEL_NORMALIZED] = True
     if source is None:
         return
-    _rewrite_request_model(request_data, request, source)
-
-
-def _rewrite_request_model(
-    request_data: dict,  # mutable-ok: the request body is rewritten in place for every downstream reader
-    request: Request | None,
-    model: str,
-) -> None:
-    request_data["model"] = model
-    _safe_set_request_parsed_body(request=request, parsed_body=request_data)
-    if request is not None:
-        request._json = request_data
-        request._body = orjson.dumps(request_data)
-
-
-_MODEL_GROUP_ALIAS_RESOLVED: Final = "litellm.model_group_alias_resolved"
+    rewrite_request_model(request_data, request, source)
 
 
 async def _resolve_router_settings_model_group_alias(
@@ -264,13 +251,16 @@ async def _resolve_router_settings_model_group_alias(
     """Rewrite the requested model through the key's or team's ``router_settings.model_group_alias``
     before the allowlist checks, so they authorize the model group the request is routed to.
     """
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers
     from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj
 
     if request is None or llm_router is None or not RouteChecks.is_llm_api_route(route=route):
         return
-    if request.scope.get(_MODEL_GROUP_ALIAS_RESOLVED) is True:
+    if request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is True:
+        return
+    request.scope[MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY] = True
+    if InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route):
         return
-    request.scope[_MODEL_GROUP_ALIAS_RESOLVED] = True
     requested: Final = request_data.get("model")
     if not isinstance(requested, str) or await read_raw_json_body(request=request) is None:
         return
@@ -284,7 +274,7 @@ async def _resolve_router_settings_model_group_alias(
         return
     verbose_proxy_logger.debug("router_settings.model_group_alias resolved %s -> %s before auth", requested, target)
     request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested)
-    _rewrite_request_model(request_data, request, target)
+    rewrite_request_model(request_data, request, target)
 
 
 def _get_model_names_for_budget_checks(
diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py
index ec2e05541cb..a1dcc6e8ece 100644
--- a/litellm/proxy/common_utils/http_parsing_utils.py
+++ b/litellm/proxy/common_utils/http_parsing_utils.py
@@ -266,6 +266,24 @@ def _safe_set_request_parsed_body(
         verbose_proxy_logger.debug("Unexpected error setting request parsed body - %s", e)
 
 
+def rewrite_request_model(
+    request_data: dict,  # mutable-ok: the request body is rewritten in place for every downstream reader
+    request: Request | None,
+    model: str,
+) -> None:
+    """Point the auth-time payload, the parsed-body cache, ``request.json()`` and ``request.body()`` at ``model``.
+    The cache and raw body keep only the keys the client sent, not params auth merged into ``request_data``.
+    """
+    request_data["model"] = model
+    if request is None:
+        return
+    cached_body: Final = _safe_get_request_parsed_body(request=request)
+    body: Final = {**cached_body, "model": model} if cached_body is not None else request_data
+    _safe_set_request_parsed_body(request=request, parsed_body=body)
+    request._json = body
+    request._body = orjson.dumps(body)
+
+
 def _safe_get_request_headers(request: Request | None) -> dict:
     """
     [Non-Blocking] Safely get the request headers.
diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
index ded45f756be..f974e4d29d2 100644
--- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
+++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
@@ -8209,6 +8209,44 @@ async def test_router_settings_model_group_alias_leaves_form_bodies_alone(monkey
     assert get_client_requested_model(request) is None
 
 
+@pytest.mark.asyncio
+async def test_router_settings_model_group_alias_rewrite_keeps_query_params_out_of_body(monkeypatch):
+    """LIT-3054: auth merges query params into its own copy of the body; the rewrite must not forward them."""
+    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
+    from litellm.proxy.common_utils.http_parsing_utils import _read_request_body, populate_request_with_path_params
+
+    router = _alias_router()
+    monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
+    body = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]}
+    request = _alias_request("/v1/chat/completions", body)
+    request.scope["query_string"] = b"api-version=2024-10-21&stream=true"
+    data = populate_request_with_path_params(request_data=await _read_request_body(request), request=request)
+    assert data["api-version"] == "2024-10-21"
+    token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku"])
+    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route="/v1/chat/completions", request=request, llm_model_list=router.model_list, llm_router=router)
+    downstream = await _read_request_body(request)
+    assert downstream == {**body, "model": "claude-haiku"}
+    assert json.loads(await request.body()) == downstream
+    assert await request.json() == downstream
+
+
+@pytest.mark.asyncio
+async def test_router_settings_model_group_alias_leaves_pass_through_bodies_alone(monkeypatch):
+    """LIT-3054: pass-through routes forward the body verbatim to the provider, so auth must not rewrite it."""
+    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
+
+    router = _alias_router()
+    monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
+    data = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]}
+    route = "/anthropic/v1/messages"
+    request = _alias_request(route, data)
+    token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku", "AgentX-LLM"])
+    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route=route, request=request, llm_model_list=router.model_list, llm_router=router)
+    assert data["model"] == "AgentX-LLM"
+    assert (await request.json())["model"] == "AgentX-LLM"
+    assert get_client_requested_model(request) is None
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("target, expect_denied", [("claude-haiku", False), ("claude-sonnet", True)])
 async def test_router_settings_model_group_alias_authorizes_target_for_team(monkeypatch, target, expect_denied):

From c48723f21085312ae19df5603c8b494831538819 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:34:35 +0000
Subject: [PATCH 42/67] fix(proxy): let a db pass-through entry override the
 yaml entry on the same path

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/proxy_server.py                 |  9 +++++-
 tests/test_litellm/proxy/test_proxy_server.py | 28 +++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 7f90874e38a..8ce997eca8d 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -7081,9 +7081,16 @@ class ProxyConfig:
         ## PASS-THROUGH ENDPOINTS ##
         if "pass_through_endpoints" in _general_settings:
             db_pass_through_endpoints: Final = _general_settings["pass_through_endpoints"]
+            db_pass_through_paths: Final = frozenset(
+                endpoint.get("path") for endpoint in db_pass_through_endpoints if isinstance(endpoint, dict)
+            )
             general_settings["pass_through_endpoints"] = [
                 *db_pass_through_endpoints,
-                *(config_passthrough_endpoints or []),
+                *(
+                    endpoint
+                    for endpoint in config_passthrough_endpoints or ()
+                    if endpoint.get("path") not in db_pass_through_paths
+                ),
             ]
             await initialize_pass_through_endpoints(pass_through_endpoints=db_pass_through_endpoints)
 
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 62c34738e51..3499a1ee1bc 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -7435,6 +7435,34 @@ async def test_update_general_settings_keeps_yaml_pass_through_endpoints_next_to
         assert still_protected.value.code == "401"
 
 
+@pytest.mark.asyncio
+async def test_update_general_settings_db_pass_through_endpoint_overrides_yaml_entry_on_the_same_path():
+    """The auth check lets any matching ``auth: false`` entry through, so a DB
+    ``auth: true`` entry can only lock down a YAML-declared path if the YAML
+    entry is dropped from the merged list."""
+    from litellm.proxy._types import ProxyException
+    from litellm.proxy.proxy_server import ProxyConfig
+
+    yaml_endpoint: Final = {"path": "/v1/cuopt/request", "target": "https://example.com/post", "auth": False}
+    db_endpoint: Final = {"id": "db-1", "path": "/v1/cuopt/request", "target": "https://example.com/post", "auth": True}
+
+    request: Final = MagicMock()
+    request.url.path = "/v1/cuopt/request"
+    request.headers = {}
+    request.query_params = {}
+
+    settings: Final = patch("litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [yaml_endpoint]})  # test-quality-ok: the method reads this module global; no injection seam
+    yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", [yaml_endpoint])  # test-quality-ok: module global holding the YAML endpoints the fix merges in
+    initialize: Final = patch("litellm.proxy.proxy_server.initialize_pass_through_endpoints", AsyncMock())  # test-quality-ok: route registration needs the FastAPI app; auth is the observable here
+    master_key: Final = patch("litellm.proxy.proxy_server.master_key", "sk-master")  # test-quality-ok: a set master key is what makes a missing Authorization header a 401
+    with settings, yaml_endpoints, initialize, master_key:
+        await ProxyConfig()._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]})
+
+        with pytest.raises(ProxyException) as locked_down:
+            await user_api_key_auth(request=request, api_key=None)
+        assert locked_down.value.code == "401"
+
+
 def _fill_user_api_key_cache(cache: DualCache, count: int) -> None:
     for index in range(count):
         cache.set_cache(key=f"key-{index}", value={"token": f"key-{index}"}, local_only=True)

From ebb9a3ceebd9e7ec14eae3b4d05c38131326fa5a Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:34:59 +0000
Subject: [PATCH 43/67] chore(ui): regenerate schema.d.ts for the /nvidia_nim
 passthrough route

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ui/litellm-dashboard/src/lib/http/schema.d.ts | 211 ++++++++++++++++++
 1 file changed, 211 insertions(+)

diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 1d6d37360ee..b787a1f1068 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -9676,6 +9676,62 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/nvidia_nim/{endpoint}": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /**
+         * Nvidia Nim Proxy Route
+         * @description Relay a native NVIDIA NIM request through a LiteLLM model group.
+         *
+         *     `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's
+         *     `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through
+         *     virtual key auth, model access checks, and spend logging.
+         */
+        get: operations["nvidia_nim_proxy_route_nvidia_nim__endpoint__get"];
+        /**
+         * Nvidia Nim Proxy Route
+         * @description Relay a native NVIDIA NIM request through a LiteLLM model group.
+         *
+         *     `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's
+         *     `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through
+         *     virtual key auth, model access checks, and spend logging.
+         */
+        put: operations["nvidia_nim_proxy_route_nvidia_nim__endpoint__put"];
+        /**
+         * Nvidia Nim Proxy Route
+         * @description Relay a native NVIDIA NIM request through a LiteLLM model group.
+         *
+         *     `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's
+         *     `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through
+         *     virtual key auth, model access checks, and spend logging.
+         */
+        post: operations["nvidia_nim_proxy_route_nvidia_nim__endpoint__post"];
+        /**
+         * Nvidia Nim Proxy Route
+         * @description Relay a native NVIDIA NIM request through a LiteLLM model group.
+         *
+         *     `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's
+         *     `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through
+         *     virtual key auth, model access checks, and spend logging.
+         */
+        delete: operations["nvidia_nim_proxy_route_nvidia_nim__endpoint__delete"];
+        options?: never;
+        head?: never;
+        /**
+         * Nvidia Nim Proxy Route
+         * @description Relay a native NVIDIA NIM request through a LiteLLM model group.
+         *
+         *     `{PROXY_BASE_URL}/nvidia_nim/{model_group}/v1/infer` forwards the body unchanged to the deployment's
+         *     `api_base`, so object detection and OCR NIMs whose payload carries no `model` field still go through
+         *     virtual key auth, model access checks, and spend logging.
+         */
+        patch: operations["nvidia_nim_proxy_route_nvidia_nim__endpoint__patch"];
+        trace?: never;
+    };
     "/ocr": {
         parameters: {
             query?: never;
@@ -53474,6 +53530,161 @@ export interface operations {
             };
         };
     };
+    nvidia_nim_proxy_route_nvidia_nim__endpoint__get: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                endpoint: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
+    nvidia_nim_proxy_route_nvidia_nim__endpoint__put: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                endpoint: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
+    nvidia_nim_proxy_route_nvidia_nim__endpoint__post: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                endpoint: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
+    nvidia_nim_proxy_route_nvidia_nim__endpoint__delete: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                endpoint: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
+    nvidia_nim_proxy_route_nvidia_nim__endpoint__patch: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                endpoint: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     ocr_ocr_post: {
         parameters: {
             query?: never;

From db0f06d15381f8d9b49e854731473362e4f562ef Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:35:48 +0000
Subject: [PATCH 44/67] fix(proxy): skip team model tpm accounting when key
 owns model tpm limit

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../hooks/parallel_request_limiter_v3.py      | 32 ++++++++++++--
 .../hooks/test_parallel_request_limiter_v3.py | 42 +++++++++++++++++++
 2 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py
index f35fb1e0042..ea1ef6f9196 100644
--- a/litellm/proxy/hooks/parallel_request_limiter_v3.py
+++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py
@@ -2893,6 +2893,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             return batch_limiter
         return None
 
+    def _key_owns_model_limit(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        requested_model: str,
+        rate_limit_key: Literal["model_rpm_limit", "model_tpm_limit"],
+    ) -> bool:
+        key_own_limits: Final = get_key_own_model_rate_limit(user_api_key_dict, rate_limit_key)
+        return key_own_limits is not None and key_own_limits.get(requested_model) is not None
+
     def _inherited_team_model_limit(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -2903,11 +2912,23 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         team_limit: Final = team_limits.get(requested_model) if team_limits else None
         if team_limit is None:
             return None
-        key_own_limits: Final = get_key_own_model_rate_limit(user_api_key_dict, rate_limit_key)
-        if key_own_limits and key_own_limits.get(requested_model) is not None:
+        if self._key_owns_model_limit(user_api_key_dict, requested_model, rate_limit_key):
             return None
         return team_limit
 
+    def _key_owns_model_tpm_limit_from_request_metadata(
+        self,
+        request_metadata: dict[str, Any],
+        model_group: str | None,
+    ) -> bool:
+        if model_group is None:
+            return False
+        key_view: Final = UserAPIKeyAuth(
+            metadata=request_metadata.get("user_api_key_metadata") or {},
+            model_max_budget=request_metadata.get("user_api_key_model_max_budget") or {},
+        )
+        return self._key_owns_model_limit(key_view, model_group, "model_tpm_limit")
+
     def _add_team_model_rate_limit_descriptor_from_metadata(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -4463,6 +4484,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             kwargs=kwargs,
             model_group=reconcile_model,
         )
+        charged_targets: Final = (
+            [target for target in targets if target[0] != "model_per_team"]
+            if self._key_owns_model_tpm_limit_from_request_metadata(request_metadata, reconcile_model)
+            else targets
+        )
         if reserved_tokens > 0 and total_tokens < reserved_tokens:
             verbose_proxy_logger.debug(
                 "Releasing unused TPM budget on success: reserved=%s, actual=%s, release=%s",
@@ -4472,7 +4498,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             )
         pipeline_operations.extend(
             self._build_reservation_aware_tpm_ops(
-                targets=targets,
+                targets=charged_targets,
                 reserved_scopes=reserved_scopes,
                 actual_tokens=total_tokens,
                 reserved_tokens=reserved_tokens,
diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
index c2b88776dc5..8d7ab89f354 100644
--- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
+++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py
@@ -6597,3 +6597,45 @@ async def test_key_model_rpm_override_keeps_team_model_tpm_limit(key_limits, ove
     assert exc.value.status_code == 429
     assert "model_per_team" in str(exc.value.detail)
     assert exc.value.headers["rate_limit_type"] == "tokens"
+
+
+@pytest.mark.parametrize(
+    "key_metadata, charges_team_model_pool",
+    [
+        ({}, True),
+        ({"model_rpm_limit": {"test-model": 10}}, True),
+        ({"model_tpm_limit": {"test-model": 5000}}, False),
+        ({"model_tpm_limit": {"other-model": 5000}}, True),
+    ],
+    ids=["no_override", "rpm_only_override", "tpm_override", "tpm_override_on_other_model"],
+)
+def test_success_tpm_accounting_skips_team_model_pool_when_key_owns_model_tpm_limit(
+    key_metadata, charges_team_model_pool
+):
+    handler = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
+    response = ModelResponse(
+        id="team-pool-tpm",
+        object="chat.completion",
+        created=int(datetime.now().timestamp()),
+        model="test-model",
+        usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150),
+        choices=[],
+    )
+    kwargs = {
+        "standard_logging_object": {"metadata": {"user_api_key_hash": hash_token("sk-pool"), "user_api_key_team_id": "t"}},
+        "litellm_params": {
+            "metadata": {
+                "model_group": "test-model",
+                "user_api_key_metadata": key_metadata,
+                "user_api_key_team_metadata": {"model_tpm_limit": {"test-model": 500}},
+            }
+        },
+        "model": "test-model",
+    }
+
+    ops = handler._build_success_event_pipeline_operations(kwargs=kwargs, response_obj=response, rate_limit_type="output")
+
+    charged_keys = {op["key"] for op in ops}
+    assert handler.create_rate_limit_keys("model_per_key", f"{hash_token('sk-pool')}:test-model", "tokens") in charged_keys
+    team_pool_key = handler.create_rate_limit_keys("model_per_team", "t:test-model", "tokens")
+    assert (team_pool_key in charged_keys) is charges_team_model_pool

From b13cf21fd10142b848130ddd2876a1d023a60ab1 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:06:05 +0000
Subject: [PATCH 45/67] fix(vertex_ai): bill Gemini Omni Interactions usage and
 Veo sampleCount on passthrough

Route Vertex and Gemini /interactions create responses through the shared
Interactions usage transformation so SpendLogs carry per-modality tokens and
cost instead of zeros, and carry parameters.sampleCount into Veo usage as
video_count so duration-based video cost is multiplied by the number of
generated videos

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/cost_calculator.py                    | 20 +++--
 litellm/llms/gemini/videos/transformation.py  |  4 +
 .../llms/vertex_ai/videos/transformation.py   | 13 ++-
 .../gemini_passthrough_logging_handler.py     | 14 +++
 .../vertex_passthrough_logging_handler.py     | 73 ++++++++++++++-
 .../pass_through_endpoints/success_handler.py | 12 ++-
 .../test_gemini_video_transformation.py       | 19 ++++
 ...test_vertex_passthrough_logging_handler.py | 90 ++++++++++++++++++-
 .../test_vertex_video_transformation.py       | 27 ++++++
 ...test_gemini_passthrough_logging_handler.py | 51 ++++++++++-
 .../test_pass_through_endpoints.py            | 20 +++++
 tests/test_litellm/test_cost_calculator.py    | 25 ++++++
 12 files changed, 356 insertions(+), 12 deletions(-)

diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 852713595d5..f81eb1539fd 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -1460,18 +1460,23 @@ def completion_cost(
                     duration_seconds: float | None = None
                     video_resolution: str | None = None
                     provider_reported_cost: float | None = None
+                    video_count: int = 1
                     if completion_response is not None and usage_obj:
                         # Handle both dict and Pydantic Usage object
                         if isinstance(usage_obj, dict):
                             duration_seconds = usage_obj.get("duration_seconds", None)
                             _vr = usage_obj.get("video_resolution", None)
                             provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
+                            _vc = usage_obj.get("video_count", None)
                         else:
                             duration_seconds = getattr(usage_obj, "duration_seconds", None)
                             _vr = getattr(usage_obj, "video_resolution", None)
                             provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
+                            _vc = getattr(usage_obj, "video_count", None)
                         if _vr is not None:
                             video_resolution = str(_vr).strip().lower()
+                        if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1:
+                            video_count = _vc
 
                         if _video_model_info is None and provider_reported_cost is not None:
                             return float(provider_reported_cost)
@@ -1482,12 +1487,15 @@ def completion_cost(
                                 video_generation_cost,
                             )
 
-                            return video_generation_cost(
-                                model=model,
-                                duration_seconds=duration_seconds,
-                                custom_llm_provider=custom_llm_provider,
-                                model_info=_video_model_info,
-                                video_resolution=video_resolution,
+                            return (
+                                video_generation_cost(
+                                    model=model,
+                                    duration_seconds=duration_seconds,
+                                    custom_llm_provider=custom_llm_provider,
+                                    model_info=_video_model_info,
+                                    video_resolution=video_resolution,
+                                )
+                                * video_count
                             )
                     # Fallback to default video cost calculation if no duration available
                     return default_video_cost_calculator(
diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py
index ff4c675b02f..a44717eb659 100644
--- a/litellm/llms/gemini/videos/transformation.py
+++ b/litellm/llms/gemini/videos/transformation.py
@@ -9,6 +9,7 @@ import litellm
 from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
 from litellm.images.utils import ImageEditRequestUtils
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
+from litellm.llms.vertex_ai.videos.transformation import veo_video_count_from_parameters
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.gemini import (
     GeminiLongRunningOperationResponse,
@@ -354,6 +355,9 @@ class GeminiVideoConfig(BaseVideoConfig):
             video_resolution: Final = _usage_video_resolution_from_parameters(parameters)
             if video_resolution is not None:
                 usage_data["video_resolution"] = video_resolution
+            video_count: Final = veo_video_count_from_parameters(parameters)
+            if video_count is not None:
+                usage_data["video_count"] = video_count
 
         video_obj.usage = usage_data
         return video_obj
diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py
index c66ad8e38b0..6525c6fa235 100644
--- a/litellm/llms/vertex_ai/videos/transformation.py
+++ b/litellm/llms/vertex_ai/videos/transformation.py
@@ -70,10 +70,18 @@ def _parse_veo_operation(raw_response: httpx.Response) -> _VeoOperation:
     return operation
 
 
+def veo_video_count_from_parameters(parameters: Mapping[str, object]) -> int | None:
+    """Number of videos Veo generates for one request (``parameters.sampleCount``)."""
+    sample_count: Final = parameters.get("sampleCount")
+    if isinstance(sample_count, bool) or not isinstance(sample_count, int) or sample_count < 1:
+        return None
+    return sample_count
+
+
 def _build_vertex_video_usage_from_request_data(
     request_data: dict[str, Any] | None,
 ) -> dict[str, float | str]:
-    """Build usage metadata (duration, resolution) for video cost calculation."""
+    """Build usage metadata (duration, resolution, video count) for video cost calculation."""
     usage_data: Final[dict[str, float | str]] = {}
     if not request_data:
         return usage_data
@@ -88,6 +96,9 @@ def _build_vertex_video_usage_from_request_data(
     res: Final = parameters.get("resolution")
     if res is not None and str(res).strip() != "":
         usage_data["video_resolution"] = str(res).strip().lower()
+    video_count: Final = veo_video_count_from_parameters(parameters)
+    if video_count is not None:
+        usage_data["video_count"] = video_count
     return usage_data
 
 
diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py
index 64d8b2929b6..a95ee87fd31 100644
--- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py
+++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py
@@ -12,6 +12,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
     ModelResponseIterator as GeminiModelResponseIterator,
 )
 from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
+from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import (
+    VertexPassthroughLoggingHandler,
+)
 from litellm.types.utils import (
     ModelResponse,
     TextCompletionResponse,
@@ -40,6 +43,17 @@ class GeminiPassthroughLoggingHandler:
         request_body: dict,
         **kwargs,
     ) -> PassThroughEndpointLoggingTypedDict:
+        if VertexPassthroughLoggingHandler.is_interactions_route(url_route):
+            return VertexPassthroughLoggingHandler.interactions_passthrough_handler(
+                httpx_response=httpx_response,
+                request_body=request_body,
+                logging_obj=logging_obj,
+                kwargs=kwargs,
+                start_time=start_time,
+                end_time=end_time,
+                custom_llm_provider="gemini",
+                vertex_location=None,
+            )
         if "predictLongRunning" in url_route:
             model = GeminiPassthroughLoggingHandler.extract_model_from_url(url_route)
 
diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
index 119a53c2411..0c4bc2a5181 100644
--- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
+++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
@@ -1,7 +1,8 @@
 import asyncio
 import re
+from collections.abc import Mapping
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, cast
 from urllib.parse import urlparse
 
 import httpx
@@ -10,6 +11,9 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    InteractionsUsageObjectTransformation,
+)
 from litellm.llms.vertex_ai.common_utils import (
     get_vertex_ai_lyria_generation_cost,
     get_vertex_location_from_url,
@@ -50,7 +54,63 @@ else:
 EndpointType = Any
 
 
+def _interactions_model(
+    response_body: Mapping[str, object],
+    request_body: Mapping[str, object] | None,
+) -> str | None:
+    response_model: Final = response_body.get("model")
+    if isinstance(response_model, str) and response_model:
+        return response_model
+    request_model: Final = (request_body or {}).get("model")
+    if isinstance(request_model, str) and request_model:
+        return request_model
+    return None
+
+
 class VertexPassthroughLoggingHandler:
+    @staticmethod
+    def is_interactions_route(url_route: str) -> bool:
+        return urlparse(url_route).path.rstrip("/").endswith("/interactions")
+
+    @staticmethod
+    def interactions_passthrough_handler(
+        httpx_response: httpx.Response,
+        request_body: Mapping[str, object] | None,
+        logging_obj: LiteLLMLoggingObj,
+        kwargs: dict,
+        start_time: datetime,
+        end_time: datetime,
+        custom_llm_provider: Literal["vertex_ai", "gemini"],
+        vertex_location: str | None,
+    ) -> PassThroughEndpointLoggingTypedDict:
+        response_body: Final[Mapping[str, object]] = httpx_response.json()
+        usage_object: Final = response_body.get("usage")
+        model: Final = _interactions_model(response_body, request_body)
+        if model is None or not InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_object):
+            return {"result": None, "kwargs": kwargs}
+
+        litellm_model_response: Final = ModelResponse(
+            model=model,
+            usage=InteractionsUsageObjectTransformation.transform_interactions_usage_object(
+                cast(Mapping[str, Any], usage_object)
+            ),
+        )
+        logging_kwargs: Final = (
+            VertexPassthroughLoggingHandler._create_vertex_response_logging_payload_for_generate_content(
+                litellm_model_response=litellm_model_response,
+                model=model,
+                kwargs=kwargs,
+                start_time=start_time,
+                end_time=end_time,
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                vertex_location=vertex_location,
+            )
+        )
+        logging_kwargs["custom_llm_provider"] = custom_llm_provider
+        logging_obj.custom_llm_provider = custom_llm_provider
+        return {"result": litellm_model_response, "kwargs": logging_kwargs}
+
     @staticmethod
     def vertex_passthrough_handler(
         httpx_response: httpx.Response,
@@ -66,6 +126,17 @@ class VertexPassthroughLoggingHandler:
         vertex_location: Final = get_vertex_location_from_url(url_route)
         if vertex_location is not None:
             logging_obj.optional_params["vertex_location"] = vertex_location
+        if VertexPassthroughLoggingHandler.is_interactions_route(url_route):
+            return VertexPassthroughLoggingHandler.interactions_passthrough_handler(
+                httpx_response=httpx_response,
+                request_body=request_body,
+                logging_obj=logging_obj,
+                kwargs=kwargs,
+                start_time=start_time,
+                end_time=end_time,
+                custom_llm_provider="vertex_ai",
+                vertex_location=vertex_location,
+            )
         if "predictLongRunning" in url_route:
             model = VertexPassthroughLoggingHandler.extract_model_from_url(url_route)
 
diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py
index c38566375f4..dd8bd17eee9 100644
--- a/litellm/proxy/pass_through_endpoints/success_handler.py
+++ b/litellm/proxy/pass_through_endpoints/success_handler.py
@@ -361,7 +361,11 @@ class PassThroughEndpointLogging:
     def is_vertex_route(self, url_route: str) -> bool:
         if any(f":{method}" in url_route for method in self.TRACKED_VERTEX_METHOD_ROUTES):
             return True
-        return any(resource in url_route for resource in self.TRACKED_VERTEX_RESOURCE_ROUTES)
+        if any(resource in url_route for resource in self.TRACKED_VERTEX_RESOURCE_ROUTES):
+            return True
+        return "/locations/" in urlparse(url_route).path and VertexPassthroughLoggingHandler.is_interactions_route(
+            url_route
+        )
 
     def is_anthropic_route(self, url_route: str):
         for route in self.TRACKED_ANTHROPIC_ROUTES:
@@ -434,8 +438,12 @@ class PassThroughEndpointLogging:
 
     def is_gemini_route(self, url_route: str, custom_llm_provider: str | None = None):
         """Check if the URL route is a Gemini API route."""
+        if custom_llm_provider != "gemini":
+            return False
+        if VertexPassthroughLoggingHandler.is_interactions_route(url_route):
+            return True
         for route in self.TRACKED_GEMINI_ROUTES:
-            if route in url_route and custom_llm_provider == "gemini":
+            if route in url_route:
                 return True
         return False
 
diff --git a/tests/test_litellm/llms/gemini/videos/test_gemini_video_transformation.py b/tests/test_litellm/llms/gemini/videos/test_gemini_video_transformation.py
index 6f215deed4e..1ac451d17db 100644
--- a/tests/test_litellm/llms/gemini/videos/test_gemini_video_transformation.py
+++ b/tests/test_litellm/llms/gemini/videos/test_gemini_video_transformation.py
@@ -430,6 +430,25 @@ class TestGeminiVideoConfig:
         assert result.usage["video_resolution"] == "1080p"
         assert result.usage["duration_seconds"] == 8.0
 
+    def test_transform_video_create_response_usage_includes_video_count(self):
+        """Regression for LIT-6896: sampleCount (number of generated videos) is copied into usage for billing."""
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.json.return_value = {"name": "operations/generate_1234567890"}
+        request_data = {
+            "instances": [{"prompt": "Test"}],
+            "parameters": {"durationSeconds": 8, "sampleCount": 3},
+        }
+        result = self.config.transform_video_create_response(
+            model="gemini/veo-3.1-fast-generate-preview",
+            raw_response=mock_response,
+            logging_obj=self.mock_logging_obj,
+            custom_llm_provider="gemini",
+            request_data=request_data,
+        )
+        assert result.usage is not None
+        assert result.usage["video_count"] == 3
+        assert result.usage["duration_seconds"] == 8.0
+
     def test_transform_video_create_response_cost_tracking_with_different_durations(
         self,
     ):
diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py
index 98010021bca..a9c5e94389c 100644
--- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py
+++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py
@@ -9,7 +9,95 @@ import litellm
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import (
     VertexPassthroughLoggingHandler,
 )
-from litellm.types.utils import PassthroughCallTypes
+from litellm.types.utils import ModelResponse, PassthroughCallTypes
+
+_OMNI_INTERACTIONS_USAGE: Final = {
+    "total_tokens": 4041,
+    "total_input_tokens": 12,
+    "input_tokens_by_modality": [{"modality": "text", "tokens": 12}],
+    "total_output_tokens": 4009,
+    "output_tokens_by_modality": [
+        {"modality": "text", "tokens": 9},
+        {"modality": "video", "tokens": 4000},
+    ],
+    "total_tool_use_tokens": 0,
+    "total_thought_tokens": 20,
+}
+
+
+def test_interactions_create_response_logs_modality_usage_and_cost() -> None:
+    """
+    Regression for LIT-6896: gemini-omni Interactions passthrough rows were logged
+    with zero tokens and zero spend. Input, text-output and video-output tokens
+    must land in usage, priced with the model's per-modality rates, and the
+    response id must stay the litellm_call_id so SpendLogs keep their request_id.
+    """
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    logging_obj.optional_params = {}
+    logging_obj.litellm_call_id = "call-6896"
+    response = httpx.Response(
+        status_code=200,
+        json={
+            "id": "interactions/abc",
+            "model": "gemini-omni-flash-preview",
+            "status": "completed",
+            "outputs": [{"type": "text", "text": "hi"}],
+            "usage": _OMNI_INTERACTIONS_USAGE,
+        },
+    )
+
+    result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
+        httpx_response=response,
+        logging_obj=logging_obj,
+        url_route="https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/interactions",
+        result=response.text,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"model": "gemini-omni-flash-preview", "input": [{"type": "text", "text": "say hi"}]},
+    )
+
+    model_response = result["result"]
+    assert isinstance(model_response, ModelResponse)
+    assert model_response.id == "call-6896"
+    usage = model_response.usage
+    assert usage.prompt_tokens == 12
+    assert usage.completion_tokens == 4009 + 20
+    assert usage.completion_tokens_details.text_tokens == 9
+    assert usage.completion_tokens_details.video_tokens == 4000
+
+    model_info = litellm.get_model_info(model="gemini-omni-flash-preview", custom_llm_provider="vertex_ai")
+    expected_cost = (
+        12 * model_info["input_cost_per_token"]
+        + (9 + 20) * model_info["output_cost_per_token"]
+        + 4000 * model_info["output_cost_per_video_token"]
+    )
+    assert result["kwargs"]["response_cost"] == pytest.approx(expected_cost)
+    assert result["kwargs"]["custom_llm_provider"] == "vertex_ai"
+    assert logging_obj.model_call_details["model"] == "gemini-omni-flash-preview"
+    assert logging_obj.model_call_details["custom_llm_provider"] == "vertex_ai"
+
+
+def test_interactions_response_without_usage_falls_back_to_generic_logging() -> None:
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    logging_obj.optional_params = {}
+    response = httpx.Response(status_code=200, json={"id": "interactions/abc", "status": "in_progress"})
+
+    result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
+        httpx_response=response,
+        logging_obj=logging_obj,
+        url_route="https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/interactions",
+        result=response.text,
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"agent": "projects/p/locations/global/reasoningEngines/1"},
+    )
+
+    assert result["result"] is None
+    assert "response_cost" not in result["kwargs"]
 
 
 def test_lyria_predict_response_preserves_audio_response_and_logs_cost(
diff --git a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py
index 04e46eab1b7..c192d22b3b7 100644
--- a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py
+++ b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py
@@ -717,6 +717,33 @@ class TestVertexAIVideoConfig:
         assert video_obj.usage["duration_seconds"] == 8.0
         assert video_obj.usage["video_resolution"] == "1080p"
 
+    @pytest.mark.parametrize(
+        "sample_count,expected_video_count",
+        [(2, 2), (1, 1), (None, None), (0, None), ("2", None)],
+        ids=["two", "one", "unset", "zero", "string"],
+    )
+    def test_transform_video_create_response_usage_includes_video_count(self, sample_count, expected_video_count):
+        """Regression for LIT-6896: sampleCount is the number of generated videos and must reach usage for billing."""
+        mock_response = Mock(spec=httpx.Response)
+        mock_response.json.return_value = {
+            "name": "projects/p/locations/us-central1/publishers/google/models/veo-3.1-fast-generate-001/operations/op-1"
+        }
+        parameters = {"durationSeconds": 4, "resolution": "720p"}
+        if sample_count is not None:
+            parameters["sampleCount"] = sample_count
+
+        video_obj = self.config.transform_video_create_response(
+            model="veo-3.1-fast-generate-001",
+            raw_response=mock_response,
+            logging_obj=self.mock_logging_obj,
+            custom_llm_provider="vertex_ai",
+            request_data={"instances": [{"prompt": "a red ball"}], "parameters": parameters},
+        )
+
+        assert video_obj.usage is not None
+        assert video_obj.usage["duration_seconds"] == 4.0
+        assert video_obj.usage.get("video_count") == expected_video_count
+
     def test_transform_video_remix_request_not_supported(self):
         """Test that video remix raises NotImplementedError."""
         with pytest.raises(NotImplementedError, match="Video remix is not supported"):
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py
index 61d1caacb91..b89ae530d6f 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_gemini_passthrough_logging_handler.py
@@ -5,7 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import httpx
 import pytest
 
-
+import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.gemini_passthrough_logging_handler import (
     GeminiPassthroughLoggingHandler,
@@ -397,3 +397,52 @@ class TestGeminiPassthroughLoggingHandler:
         assert mock_logging_obj.model_call_details["response_cost"] == expected_cost
         assert mock_logging_obj.model_call_details["model"] == "veo-2.0-generate-001"
         assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini"
+
+    def test_interactions_create_response_is_priced_as_gemini(self):
+        """Regression for LIT-6896: Gemini API Interactions passthrough must not log zero usage."""
+        usage = {
+            "total_tokens": 1030,
+            "total_input_tokens": 10,
+            "input_tokens_by_modality": [{"modality": "text", "tokens": 10}],
+            "total_output_tokens": 1020,
+            "output_tokens_by_modality": [
+                {"modality": "text", "tokens": 20},
+                {"modality": "video", "tokens": 1000},
+            ],
+            "total_tool_use_tokens": 0,
+            "total_thought_tokens": 0,
+        }
+        mock_httpx_response = MagicMock(spec=httpx.Response)
+        mock_httpx_response.json.return_value = {
+            "id": "interactions/abc",
+            "model": "gemini-omni-flash-preview",
+            "status": "completed",
+            "usage": usage,
+        }
+        mock_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
+        mock_logging_obj.model_call_details = {}
+        mock_logging_obj.litellm_call_id = "call-6896"
+
+        result = GeminiPassthroughLoggingHandler.gemini_passthrough_handler(
+            httpx_response=mock_httpx_response,
+            response_body=mock_httpx_response.json.return_value,
+            logging_obj=mock_logging_obj,
+            url_route="https://generativelanguage.googleapis.com/v1beta/interactions",
+            result="",
+            start_time=self.start_time,
+            end_time=self.end_time,
+            cache_hit=False,
+            request_body={"model": "gemini-omni-flash-preview", "input": "make a clip"},
+        )
+
+        model_info = litellm.get_model_info(model="gemini-omni-flash-preview", custom_llm_provider="gemini")
+        expected_cost = (
+            10 * model_info["input_cost_per_token"]
+            + 20 * model_info["output_cost_per_token"]
+            + 1000 * model_info["output_cost_per_video_token"]
+        )
+        assert result["result"].id == "call-6896"
+        assert result["result"].usage.completion_tokens_details.video_tokens == 1000
+        assert result["kwargs"]["response_cost"] == pytest.approx(expected_cost)
+        assert result["kwargs"]["custom_llm_provider"] == "gemini"
+        assert mock_logging_obj.model_call_details["custom_llm_provider"] == "gemini"
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
index 11066d4ed38..988c4782f6c 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
@@ -496,6 +496,26 @@ def test_is_vertex_route_ignores_plain_predict_path_segment():
     )
 
 
+def test_interactions_create_routes_are_tracked_for_vertex_and_gemini():
+    """
+    Regression for LIT-6896: Interactions API (gemini-omni) passthrough responses
+    were never handed to the Vertex/Gemini logging handlers, so SpendLogs rows
+    landed with zero tokens and zero spend. Only the create URL is billable;
+    GET/DELETE on an interaction id and non-Google `/interactions` URLs stay generic.
+    """
+    handler = PassThroughEndpointLogging()
+    vertex_create = "https://aiplatform.googleapis.com/v1beta1/projects/p/locations/global/interactions"
+    gemini_create = "https://generativelanguage.googleapis.com/v1beta/interactions"
+
+    assert handler.is_vertex_route(vertex_create) is True
+    assert handler.is_vertex_route(f"{vertex_create}/abc123") is False
+    assert handler.is_vertex_route("https://upstream.example.com/api/interactions") is False
+
+    assert handler.is_gemini_route(gemini_create, custom_llm_provider="gemini") is True
+    assert handler.is_gemini_route(f"{gemini_create}/abc123", custom_llm_provider="gemini") is False
+    assert handler.is_gemini_route(gemini_create, custom_llm_provider=None) is False
+
+
 @pytest.mark.asyncio
 async def test_custom_passthrough_predict_path_logs_via_generic_handler():
     """
diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py
index ef797ef8bcc..f7ea2c00891 100644
--- a/tests/test_litellm/test_cost_calculator.py
+++ b/tests/test_litellm/test_cost_calculator.py
@@ -4141,6 +4141,31 @@ def test_completion_cost_bills_interactions_video_output_at_video_rate():
     assert cost == pytest.approx(expected)
 
 
+@pytest.mark.parametrize("video_count", [2, 3])
+def test_completion_cost_multiplies_video_cost_by_generated_video_count(video_count: int) -> None:
+    """Regression for LIT-6896: a Veo request for N samples generates N videos and must be billed N times."""
+    from litellm.types.videos.main import VideoObject
+
+    def _video(usage: dict[str, object]) -> VideoObject:
+        return VideoObject(id="v", object="video", status="processing", model="veo-3.1-fast-generate-001", usage=usage)
+
+    single_cost = completion_cost(
+        completion_response=_video({"duration_seconds": 4.0, "video_resolution": "720p"}),
+        model="veo-3.1-fast-generate-001",
+        custom_llm_provider="vertex_ai",
+        call_type="create_video",
+    )
+    multi_cost = completion_cost(
+        completion_response=_video({"duration_seconds": 4.0, "video_resolution": "720p", "video_count": video_count}),
+        model="veo-3.1-fast-generate-001",
+        custom_llm_provider="vertex_ai",
+        call_type="create_video",
+    )
+
+    assert single_cost > 0
+    assert multi_cost == pytest.approx(single_cost * video_count)
+
+
 @pytest.mark.parametrize(
     "batch_rate,expected_prompt,expected_completion",
     [

From 65132ef2e92ea1afbe6bed1813404a27e59772f7 Mon Sep 17 00:00:00 2001
From: shivam <shivam@berri.ai>
Date: Tue, 15 Sep 2026 22:42:58 +0000
Subject: [PATCH 46/67] fix(cost): price native Responses WebSocket turns at
 their returned service_tier

The logging object built for aresponses_websocket sessions dropped the
service_tier carried by each billable response.completed/response.incomplete
event, so sessions were priced at the default tier. The logging object now
carries the tier when the session is single-tier, and completion_cost splits
mixed-tier sessions into one object per tier before pricing. Fixes #41299

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/cost_calculator.py                    | 85 +++++++++++++++++--
 litellm/litellm_core_utils/litellm_logging.py |  5 ++
 litellm/types/utils.py                        |  1 +
 .../test_litellm_logging.py                   | 60 ++++++++++++-
 tests/test_litellm/test_cost_calculator.py    | 77 ++++++++++++++++-
 5 files changed, 219 insertions(+), 9 deletions(-)

diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 852713595d5..fca2172886a 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -1203,6 +1203,24 @@ def _without_provider_stated_cost(usage: Usage | None) -> Usage | None:
     return usage.model_copy(update=MappingProxyType({"cost": None}))
 
 
+def _split_responses_ws_logging_object_by_service_tier(
+    completion_response: LiteLLMRealtimeStreamLoggingObject,
+) -> tuple[LiteLLMRealtimeStreamLoggingObject, ...] | None:
+    partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(
+        cast(Sequence[Mapping[str, object]], completion_response.results)
+    )
+    if len(partition) <= 1:
+        return None
+    return tuple(
+        LiteLLMRealtimeStreamLoggingObject(
+            results=cast(OpenAIRealtimeStreamList, list(group)),
+            usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(group),
+            service_tier=tier,
+        )
+        for tier, group in partition.items()
+    )
+
+
 def completion_cost(
     completion_response: object | None = None,
     model: str | None = None,
@@ -1266,6 +1284,41 @@ def completion_cost(
     try:
         call_type = _infer_call_type(call_type, completion_response) or "completion"
 
+        if call_type == CallTypes.aresponses_websocket.value and isinstance(
+            completion_response, LiteLLMRealtimeStreamLoggingObject
+        ):
+            ws_tier_parts: Final = _split_responses_ws_logging_object_by_service_tier(completion_response)
+            if ws_tier_parts is not None:
+                return sum(
+                    completion_cost(
+                        completion_response=part,
+                        model=model,
+                        prompt=prompt,
+                        messages=messages,
+                        completion=completion,
+                        total_time=total_time,
+                        call_type=call_type,
+                        custom_llm_provider=custom_llm_provider,
+                        region_name=region_name,
+                        size=size,
+                        quality=quality,
+                        n=n,
+                        custom_cost_per_token=custom_cost_per_token,
+                        custom_cost_per_second=custom_cost_per_second,
+                        optional_params=optional_params,
+                        custom_pricing=custom_pricing,
+                        base_model=base_model,
+                        standard_built_in_tools_params=standard_built_in_tools_params,
+                        litellm_model_name=litellm_model_name,
+                        router_model_id=router_model_id,
+                        litellm_logging_obj=litellm_logging_obj,
+                        service_tier=service_tier,
+                        data_residency=data_residency,
+                        vertex_location=vertex_location,
+                    )
+                    for part in ws_tier_parts
+                )
+
         if (
             (call_type == "aimage_generation" or call_type == "image_generation")
             and model is not None
@@ -2558,6 +2611,7 @@ _RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "re
 
 class _ResponsesWsEventResponse(BaseModel):
     usage: Mapping[str, object] | None = None
+    service_tier: str | None = None
 
 
 class _ResponsesWsEvent(BaseModel):
@@ -2565,20 +2619,39 @@ class _ResponsesWsEvent(BaseModel):
     response: _ResponsesWsEventResponse | None = None
 
 
+def _billable_responses_ws_events(
+    results: Sequence[Mapping[str, object]],
+) -> tuple[tuple[Mapping[str, object], _ResponsesWsEventResponse], ...]:
+    return tuple(
+        (result, event.response)
+        for result in results
+        if (event := _ResponsesWsEvent.model_validate(result)).type in _RESPONSES_WS_BILLABLE_EVENT_TYPES
+        and event.response is not None
+        and event.response.usage is not None
+    )
+
+
 class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
     @staticmethod
     def collect_usage_from_responses_ws_results(
         results: Sequence[Mapping[str, object]],
     ) -> tuple[Usage, ...]:
-        events: Final = tuple(_ResponsesWsEvent.model_validate(result) for result in results)
         return tuple(
             ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(  # pyright: ignore[reportPrivateUsage]  # same shared transform the realtime processor uses
-                event.response.usage
+                response.usage
             )
-            for event in events
-            if event.type in _RESPONSES_WS_BILLABLE_EVENT_TYPES
-            and event.response is not None
-            and event.response.usage is not None
+            for _, response in _billable_responses_ws_events(results)
+            if response.usage is not None
+        )
+
+    @staticmethod
+    def partition_results_by_service_tier(
+        results: Sequence[Mapping[str, object]],
+    ) -> Mapping[str | None, tuple[Mapping[str, object], ...]]:
+        billable: Final = _billable_responses_ws_events(results)
+        tiers: Final = dict.fromkeys(response.service_tier for _, response in billable)
+        return MappingProxyType(
+            {tier: tuple(result for result, response in billable if response.service_tier == tier) for tier in tiers}
         )
 
     @staticmethod
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index abac624d5ec..40621a2f68d 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -2101,9 +2101,14 @@ class Logging(LiteLLMLoggingBaseClass):
                     results=result  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
                 )
             )
+            ws_tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(
+                results=result  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+            )
+            ws_service_tier: Final = next(iter(ws_tier_partition)) if len(ws_tier_partition) == 1 else None
             logging_result = LiteLLMRealtimeStreamLoggingObject(
                 usage=combined_ws_usage,
                 results=result,  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                service_tier=ws_service_tier,
             )
 
         elif (
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index fdf533fb4e9..298c30bbec2 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -4275,6 +4275,7 @@ class LiteLLMRealtimeStreamLoggingObject(LiteLLMPydanticObjectBase):
     # rate_limits.updated), blocks the event loop, and discards the session usage.
     results: SkipValidation[OpenAIRealtimeStreamList]
     usage: Usage
+    service_tier: str | None = None
     _hidden_params: dict = {}
 
     @field_serializer("results")
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index dd1ad9c9623..aaf44b8e918 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -6554,9 +6554,9 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o
     assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == ""
 
 
-def _responses_ws_logging_obj() -> LitellmLogging:
+def _responses_ws_logging_obj(model: str = "gpt-4o") -> LitellmLogging:
     return LitellmLogging(
-        model="gpt-4o",
+        model=model,
         messages=[],
         stream=False,
         call_type=CallTypes.aresponses_websocket.value,
@@ -6638,6 +6638,62 @@ def test_normalize_logging_result_bills_incomplete_responses_websocket_turns():
     assert normalized.usage.total_tokens == 75
 
 
+def test_normalize_logging_result_prices_responses_websocket_at_returned_service_tier():
+    """Issue #41299: a WebSocket turn billed at priority tier reported it on
+    response.completed.response.service_tier, but the logging object dropped it and the
+    session was priced at the default tier."""
+    events = [
+        {"type": "response.created", "response": {}},
+        {
+            "type": "response.completed",
+            "response": {
+                "service_tier": "priority",
+                "usage": {"input_tokens": 100, "output_tokens": 40, "total_tokens": 140},
+            },
+        },
+    ]
+
+    normalized = _responses_ws_logging_obj(model="gpt-5.4").normalize_logging_result(result=events)
+
+    assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject)
+    assert normalized.service_tier == "priority"
+
+    usage = ResponseAPIUsage(input_tokens=100, output_tokens=40, total_tokens=140)
+    ws_cost = litellm.completion_cost(
+        completion_response=normalized,
+        model="gpt-5.4",
+        call_type=CallTypes.aresponses_websocket.value,
+        custom_llm_provider="openai",
+    )
+    priority_http_cost = litellm.completion_cost(
+        completion_response=ResponsesAPIResponse(
+            id="resp-priority",
+            created_at=1700000000,
+            output=[],
+            service_tier="priority",
+            usage=usage,
+        ),
+        model="gpt-5.4",
+        call_type=CallTypes.aresponses.value,
+        custom_llm_provider="openai",
+    )
+    default_http_cost = litellm.completion_cost(
+        completion_response=ResponsesAPIResponse(
+            id="resp-default",
+            created_at=1700000000,
+            output=[],
+            service_tier="default",
+            usage=usage,
+        ),
+        model="gpt-5.4",
+        call_type=CallTypes.aresponses.value,
+        custom_llm_provider="openai",
+    )
+
+    assert ws_cost == priority_http_cost
+    assert priority_http_cost > default_http_cost
+
+
 def test_get_standard_logging_object_payload_reads_overhead_from_logging_obj_for_dict_results(logging_obj):
     """LIT-5466: /v1/messages returns a plain dict with no _hidden_params, so the overhead
     recorded on the logging object must reach hidden_params.litellm_overhead_time_ms (SpendLogs)."""
diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py
index ef797ef8bcc..6f5cde58ff3 100644
--- a/tests/test_litellm/test_cost_calculator.py
+++ b/tests/test_litellm/test_cost_calculator.py
@@ -1,4 +1,5 @@
 
+import time
 from typing import Final
 
 import pytest
@@ -9,6 +10,7 @@ import litellm
 from litellm.cost_calculator import (
     BaseTokenUsageProcessor,
     RealtimeAPITokenUsageProcessor,
+    ResponsesWebSocketTokenUsageProcessor,
     completion_cost,
     cost_per_token,
     handle_realtime_stream_cost_calculation,
@@ -17,10 +19,12 @@ from litellm.cost_calculator import (
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.llms.base_llm.ocr.transformation import OCRPage, OCRResponse, OCRUsageInfo
 from litellm.types.llms.base import CachedTokensDetails
-from litellm.types.llms.openai import OpenAIRealtimeStreamList
+from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse
 from litellm.types.rerank import RerankResponse
 from litellm.types.utils import (
     CacheCreationTokenDetails,
+    CallTypes,
+    LiteLLMRealtimeStreamLoggingObject,
     ModelInfo,
     ModelResponse,
     PromptTokensDetailsWrapper,
@@ -5243,3 +5247,74 @@ def test_completion_cost_ocr_ignores_deployment_pricing_without_custom_pricing_f
         litellm_logging_obj=logging_obj,
     )
     assert cost == 0.0
+
+
+def test_completion_cost_prices_responses_websocket_turns_per_service_tier():
+    """Issue #41299: a session mixing default and priority turns must price each turn at
+    its own returned service_tier, not the summed usage at a single tier."""
+    events = [
+        {"type": "response.created", "response": {}},
+        {
+            "type": "response.completed",
+            "response": {
+                "service_tier": "default",
+                "usage": {"input_tokens": 100, "output_tokens": 40, "total_tokens": 140},
+            },
+        },
+        {"type": "rate_limits.updated", "rate_limits": {}},
+        {
+            "type": "response.completed",
+            "response": {
+                "service_tier": "priority",
+                "usage": {"input_tokens": 60, "output_tokens": 10, "total_tokens": 70},
+            },
+        },
+        {"type": "response.failed", "response": {"usage": None}},
+    ]
+
+    partition = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(events)
+    assert tuple(partition.keys()) == ("default", "priority")
+    assert len(partition["default"]) == 1
+    assert len(partition["priority"]) == 1
+
+    logging_obj = Logging(
+        model="gpt-5.4",
+        messages=[],
+        stream=False,
+        call_type=CallTypes.aresponses_websocket.value,
+        start_time=time.time(),
+        litellm_call_id="responses-ws-tier-test",
+        function_id="responses-ws-tier-test",
+    )
+    normalized = logging_obj.normalize_logging_result(result=events)
+    assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject)
+    assert normalized.service_tier is None
+
+    def _http_cost(input_tokens: int, output_tokens: int, service_tier: str) -> float:
+        return completion_cost(
+            completion_response=ResponsesAPIResponse(
+                id=f"resp-{service_tier}",
+                created_at=1700000000,
+                output=[],
+                service_tier=service_tier,
+                usage=ResponseAPIUsage(
+                    input_tokens=input_tokens,
+                    output_tokens=output_tokens,
+                    total_tokens=input_tokens + output_tokens,
+                ),
+            ),
+            model="gpt-5.4",
+            call_type=CallTypes.aresponses.value,
+            custom_llm_provider="openai",
+        )
+
+    ws_cost = completion_cost(
+        completion_response=normalized,
+        model="gpt-5.4",
+        call_type=CallTypes.aresponses_websocket.value,
+        custom_llm_provider="openai",
+    )
+
+    assert ws_cost == pytest.approx(_http_cost(100, 40, "default") + _http_cost(60, 10, "priority"))
+    assert ws_cost != pytest.approx(_http_cost(160, 50, "default"))
+    assert ws_cost != pytest.approx(_http_cost(160, 50, "priority"))

From 91dde59bc32827513e41d0afc9f515be7201fd6d Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:48:47 +0000
Subject: [PATCH 47/67] refactor(proxy): validate request metadata before
 reconstructing key rate-limit view

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/hooks/parallel_request_limiter_v3.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py
index ea1ef6f9196..f72720b4726 100644
--- a/litellm/proxy/hooks/parallel_request_limiter_v3.py
+++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py
@@ -2918,14 +2918,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
     def _key_owns_model_tpm_limit_from_request_metadata(
         self,
-        request_metadata: dict[str, Any],
+        request_metadata: Mapping[str, object],
         model_group: str | None,
     ) -> bool:
         if model_group is None:
             return False
-        key_view: Final = UserAPIKeyAuth(
-            metadata=request_metadata.get("user_api_key_metadata") or {},
-            model_max_budget=request_metadata.get("user_api_key_model_max_budget") or {},
+        key_view: Final = UserAPIKeyAuth.model_validate(
+            {
+                "metadata": request_metadata.get("user_api_key_metadata") or {},
+                "model_max_budget": request_metadata.get("user_api_key_model_max_budget") or {},
+            }
         )
         return self._key_owns_model_limit(key_view, model_group, "model_tpm_limit")
 

From ad74c3ede75aaf3dadbe98864849372e0741f70b Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 15 Sep 2026 15:48:54 -0700
Subject: [PATCH 48/67] bump: litellm-enterprise 0.1.67 -> 0.1.68,
 litellm-proxy-extras 0.4.97 -> 0.4.98, litellm 1.102.0 -> 1.103.0

---
 enterprise/pyproject.toml           | 4 ++--
 litellm-proxy-extras/pyproject.toml | 4 ++--
 pyproject.toml                      | 8 ++++----
 uv.lock                             | 8 ++++----
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index c049bf68c46..06b1da7ea76 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.67"
+version = "0.1.68"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.67"
+version = "0.1.68"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",
diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index f94591872a4..914b9c5a14b 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.97"
+version = "0.4.98"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.97"
+version = "0.4.98"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",
diff --git a/pyproject.toml b/pyproject.toml
index bdc0e09a17f..e8fa73fc9e1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.102.0"
+version = "1.103.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -67,8 +67,8 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.97",
-    "litellm-enterprise==0.1.67",
+    "litellm-proxy-extras==0.4.98",
+    "litellm-enterprise==0.1.68",
     "RestrictedPython>=8.5,<9.0",
     "rich>=13.9.4,<14.0",
     "InquirerPy>=0.3.4,<1.0",
@@ -331,7 +331,7 @@ members = ["enterprise", "litellm-proxy-extras"]
 profile = "black"
 
 [tool.commitizen]
-version = "1.102.0"
+version = "1.103.0"
 version_files = [
     "pyproject.toml:^version",
 ]
diff --git a/uv.lock b/uv.lock
index 68c2d07b722..08c4e4da76c 100644
--- a/uv.lock
+++ b/uv.lock
@@ -10,7 +10,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-12T03:51:49.261499386Z"
+exclude-newer = "2026-09-12T22:48:38.53978Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -4457,7 +4457,7 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.102.0"
+version = "1.103.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },
@@ -4874,12 +4874,12 @@ proxy-dev = [
 
 [[package]]
 name = "litellm-enterprise"
-version = "0.1.67"
+version = "0.1.68"
 source = { editable = "enterprise" }
 
 [[package]]
 name = "litellm-proxy-extras"
-version = "0.4.97"
+version = "0.4.98"
 source = { editable = "litellm-proxy-extras" }
 
 [[package]]

From 024a8875214f971cefc354405e40833db188f5c8 Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Tue, 15 Sep 2026 22:49:35 +0000
Subject: [PATCH 49/67] feat(model_info): add Gemini fallback generalization
 rules (vertex routing + 2.5+ family baseline)

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ...odel_prices_and_context_window_backup.json | 32 +++++++
 model_prices_and_context_window.json          | 32 +++++++
 .../test_fallback_generalizations.py          | 93 ++++++++++++++++++-
 3 files changed, 154 insertions(+), 3 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 9f91cf82f41..afe2cb3deb9 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -58149,6 +58149,38 @@
                 "model_info": {
                     "supports_reasoning": true
                 }
+            },
+            {
+                "name": "vertex-gemini-ids",
+                "pattern": "^gemini-(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?:-(?:preview|exp|latest))?(?:-\\d{2}-\\d{2,4})?$",
+                "description": "A bare Gemini text-chat id at 2.5 or higher, anchored to the whole name: gemini-<major>[.minor]-(pro|flash)[-lite] with an optional preview/exp/latest tag and an optional MM-YYYY or MM-DD date. Bare Gemini ids in this map already infer vertex_ai (gemini-2.5-pro is filed under vertex_ai-language-models), so an unmapped id such as gemini-4-pro follows the same convention instead of raising 'LLM Provider NOT provided'. Anchored so a namespaced or non-chat id (gemini/..., -image, -tts) stays on its own path.",
+                "model_info": {
+                    "litellm_provider": "vertex_ai"
+                }
+            },
+            {
+                "name": "gemini-family-baseline",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics))(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with a 1M context and 64K output; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
+                "model_info": {
+                    "mode": "chat",
+                    "max_input_tokens": 1048576,
+                    "max_output_tokens": 65536,
+                    "max_tokens": 65536,
+                    "supports_function_calling": true,
+                    "supports_parallel_function_calling": true,
+                    "supports_vision": true,
+                    "supports_pdf_input": true,
+                    "supports_audio_input": true,
+                    "supports_video_input": true,
+                    "supports_tool_choice": true,
+                    "supports_system_messages": true,
+                    "supports_response_schema": true,
+                    "supports_prompt_caching": true,
+                    "supports_reasoning": true,
+                    "supports_web_search": true,
+                    "supports_url_context": true
+                }
             }
         ]
     },
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 9f91cf82f41..afe2cb3deb9 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -58149,6 +58149,38 @@
                 "model_info": {
                     "supports_reasoning": true
                 }
+            },
+            {
+                "name": "vertex-gemini-ids",
+                "pattern": "^gemini-(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?:-(?:preview|exp|latest))?(?:-\\d{2}-\\d{2,4})?$",
+                "description": "A bare Gemini text-chat id at 2.5 or higher, anchored to the whole name: gemini-<major>[.minor]-(pro|flash)[-lite] with an optional preview/exp/latest tag and an optional MM-YYYY or MM-DD date. Bare Gemini ids in this map already infer vertex_ai (gemini-2.5-pro is filed under vertex_ai-language-models), so an unmapped id such as gemini-4-pro follows the same convention instead of raising 'LLM Provider NOT provided'. Anchored so a namespaced or non-chat id (gemini/..., -image, -tts) stays on its own path.",
+                "model_info": {
+                    "litellm_provider": "vertex_ai"
+                }
+            },
+            {
+                "name": "gemini-family-baseline",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics))(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with a 1M context and 64K output; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
+                "model_info": {
+                    "mode": "chat",
+                    "max_input_tokens": 1048576,
+                    "max_output_tokens": 65536,
+                    "max_tokens": 65536,
+                    "supports_function_calling": true,
+                    "supports_parallel_function_calling": true,
+                    "supports_vision": true,
+                    "supports_pdf_input": true,
+                    "supports_audio_input": true,
+                    "supports_video_input": true,
+                    "supports_tool_choice": true,
+                    "supports_system_messages": true,
+                    "supports_response_schema": true,
+                    "supports_prompt_caching": true,
+                    "supports_reasoning": true,
+                    "supports_web_search": true,
+                    "supports_url_context": true
+                }
             }
         ]
     },
diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
index 057fa228562..e91a2bb4394 100644
--- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
+++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
@@ -135,9 +135,7 @@ def test_fill_missing_requires_per_rule_opt_in(restore_generalizations):
         "supports_vision": True,
     }
 
-    restore_generalizations(
-        [{"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}]
-    )
+    restore_generalizations([{"name": "base", "pattern": r"^acme-", "model_info": {"supports_reasoning": True}}])
     assert match_fill_missing_generalizations("acme-1", "openai") is None
 
     restore_generalizations(
@@ -451,6 +449,95 @@ def shipped_cost_map(monkeypatch):
         set_fallback_generalizations(previous_rules)
 
 
+def test_shipped_bare_gemini_id_routes_to_vertex_ai(shipped_cost_map):
+    for model in (
+        "gemini-3.9-flash",
+        "gemini-4-pro",
+        "gemini-4-pro-preview",
+        "gemini-3.9-flash-lite-preview-09-2026",
+    ):
+        assert model not in litellm.model_cost
+        assert litellm.get_llm_provider(model=model)[1] == "vertex_ai"
+
+    assert litellm.get_llm_provider(model="gemini-2.5-pro")[1] == "vertex_ai"
+
+
+def test_shipped_gemini_routing_rule_is_anchored(shipped_cost_map):
+    for model in (
+        "gemini-4-flash-image",
+        "gemini-3.9-flash-preview-tts",
+        "gemini-2.0-flash-new",
+        "gemini-1.5-flash-new",
+        "gemini-4-flashy",
+        "gemini/gemini-4-pro",
+    ):
+        assert match_routing_generalization(model) is None, model
+
+
+@pytest.mark.parametrize(
+    "model,provider",
+    [
+        ("gemini-4-pro", "gemini"),
+        ("gemini-3.9-flash", "vertex_ai"),
+        ("databricks-gemini-4-1-pro", "databricks"),
+        ("gemini-4-pro-preview-customtools", "gemini"),
+    ],
+)
+def test_shipped_gemini_baseline_resolves_unmapped_ids_provider_neutral(shipped_cost_map, model, provider):
+    assert model not in litellm.model_cost
+    if provider == "gemini":
+        assert f"gemini/{model}" not in litellm.model_cost
+
+    info = litellm.get_model_info(model, custom_llm_provider=provider)
+    assert info["litellm_provider"] == provider
+    assert info["mode"] == "chat"
+    assert info["max_input_tokens"] == 1048576
+    assert info["max_tokens"] == 65536
+    assert info["supports_reasoning"] is True
+    assert info["supports_function_calling"] is True
+    assert info["supports_vision"] is True
+    assert not info.get("input_cost_per_token")
+    assert not info.get("output_cost_per_token")
+
+
+def test_shipped_gemini_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cost_map):
+    for model in (
+        "gemini-4-flash-image",
+        "gemini-3.9-flash-preview-tts",
+        "gemini-4-flash-live-preview",
+        "gemini-4-flash-native-audio",
+        "gemini-embedding-4",
+        "gemini-2.5-computer-use-preview-12-2026",
+        "gemini-2.0-flash-new",
+        "gemini-1.5-pro-new",
+        "gemini-4-flashy",
+    ):
+        assert match_capability_generalizations(model) is None, model
+
+
+def test_shipped_gemini_baseline_keeps_reasoning_effort_on_unmapped_model(shipped_cost_map):
+    assert litellm.supports_reasoning(model="gemini-4-pro", custom_llm_provider="gemini") is True
+
+    optional_params = litellm.utils.get_optional_params(
+        model="gemini-4-pro",
+        custom_llm_provider="gemini",
+        reasoning_effort="medium",
+        drop_params=False,
+    )
+    assert isinstance(optional_params, dict)
+    assert optional_params["thinkingConfig"]["thinkingBudget"] > 0
+    assert optional_params["thinkingConfig"]["includeThoughts"] is True
+
+
+def test_shipped_gemini_baseline_loses_to_exact_entries(shipped_cost_map):
+    model = "gemini-2.5-flash-lite"
+    info = litellm.get_model_info(model, custom_llm_provider="gemini")
+    entry = litellm.model_cost["gemini/gemini-2.5-flash-lite"]
+    assert info["max_tokens"] == entry["max_tokens"]
+    assert info["input_cost_per_token"] == entry["input_cost_per_token"]
+    assert entry["input_cost_per_token"] > 0
+
+
 def test_shipped_bare_claude_id_routes_to_anthropic(shipped_cost_map):
     _, provider, _, _ = litellm.get_llm_provider(model="claude-haiku-4-6")
     assert provider == "anthropic"

From a654034f6a61943d2b7a9335e940565f90a2ff16 Mon Sep 17 00:00:00 2001
From: shivam <shivam@berri.ai>
Date: Tue, 15 Sep 2026 22:50:28 +0000
Subject: [PATCH 50/67] test(ci): allow completion_cost recursion for
 mixed-tier WS pricing split

The recursion is bounded to depth 1: split parts each carry a single
service_tier, so the recursive call's partition has one key and the split
helper returns None.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/code_coverage_tests/recursive_detector.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py
index e9f87ba6cae..d8e318c61af 100644
--- a/tests/code_coverage_tests/recursive_detector.py
+++ b/tests/code_coverage_tests/recursive_detector.py
@@ -67,6 +67,7 @@ IGNORE_FUNCTIONS = [
     "_redact_agent_params_tree",  # max depth set (default 10), same shape as _redact_sensitive_litellm_params.
     "_restore_redacted_nested_value",  # max depth set (default 10), mirrors _redact_agent_params_tree on the write side.
     "_unqualified",  # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
+    "completion_cost",  # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
 ]
 
 

From fd90eeb3c60fd21cdee7e54ce77f394e35bfa0b9 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:50:29 +0000
Subject: [PATCH 51/67] test(proxy): cover disjoint-method db/yaml pass-through
 entries on a shared path

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 tests/test_litellm/proxy/test_proxy_server.py | 32 +++++++++++++++----
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 3499a1ee1bc..2c63443da66 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -7436,18 +7436,38 @@ async def test_update_general_settings_keeps_yaml_pass_through_endpoints_next_to
 
 
 @pytest.mark.asyncio
-async def test_update_general_settings_db_pass_through_endpoint_overrides_yaml_entry_on_the_same_path():
-    """The auth check lets any matching ``auth: false`` entry through, so a DB
-    ``auth: true`` entry can only lock down a YAML-declared path if the YAML
-    entry is dropped from the merged list."""
+@pytest.mark.parametrize(
+    ("db_methods", "yaml_methods"),
+    [(None, None), (["POST"], ["GET"])],
+    ids=["all-methods", "disjoint-methods"],
+)
+async def test_update_general_settings_db_pass_through_endpoint_overrides_yaml_entry_on_the_same_path(
+    db_methods: list[str] | None, yaml_methods: list[str] | None
+):
+    """The auth check matches pass-through entries by path only and lets any
+    matching ``auth: false`` entry through, so a DB ``auth: true`` entry can only
+    lock down a YAML-declared path if the YAML entry is dropped from the merged
+    list, whatever ``methods`` either entry declares."""
     from litellm.proxy._types import ProxyException
     from litellm.proxy.proxy_server import ProxyConfig
 
-    yaml_endpoint: Final = {"path": "/v1/cuopt/request", "target": "https://example.com/post", "auth": False}
-    db_endpoint: Final = {"id": "db-1", "path": "/v1/cuopt/request", "target": "https://example.com/post", "auth": True}
+    yaml_endpoint: Final = {
+        "path": "/v1/cuopt/request",
+        "target": "https://example.com/post",
+        "auth": False,
+        "methods": yaml_methods,
+    }
+    db_endpoint: Final = {
+        "id": "db-1",
+        "path": "/v1/cuopt/request",
+        "target": "https://example.com/post",
+        "auth": True,
+        "methods": db_methods,
+    }
 
     request: Final = MagicMock()
     request.url.path = "/v1/cuopt/request"
+    request.method = "POST"
     request.headers = {}
     request.query_params = {}
 

From b50a22b370baedefa18a77bcb8ad391bfaf5a042 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:55:14 +0000
Subject: [PATCH 52/67] fix(proxy): restrict /nvidia_nim route to NIM-backed
 model groups and inject router in tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../nvidia_nim/passthrough/transformation.py  |  15 +-
 litellm/proxy/auth/auth_utils.py              |   7 +-
 .../llm_passthrough_endpoints.py              |  28 +++-
 .../proxy/auth/test_auth_utils.py             |   5 +
 .../test_llm_pass_through_endpoints.py        | 140 ++++++++++--------
 5 files changed, 124 insertions(+), 71 deletions(-)

diff --git a/litellm/llms/nvidia_nim/passthrough/transformation.py b/litellm/llms/nvidia_nim/passthrough/transformation.py
index f6273daaabb..6df9681cc81 100644
--- a/litellm/llms/nvidia_nim/passthrough/transformation.py
+++ b/litellm/llms/nvidia_nim/passthrough/transformation.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Collection, Mapping, Sequence
+from collections.abc import Collection, Iterable, Mapping, Sequence
 from typing import TYPE_CHECKING, Final
 
 import httpx
@@ -14,7 +14,8 @@ from litellm.llms.base_llm.passthrough.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import StandardPassThroughResponseObject
+from litellm.types.router import DeploymentTypedDict
+from litellm.types.utils import LlmProviders, StandardPassThroughResponseObject
 
 if TYPE_CHECKING:
     from httpx import URL, Response
@@ -25,6 +26,16 @@ if TYPE_CHECKING:
 
 
 API_VERSION_SEGMENT: Final = re.compile(r"^v\d+$")
+NVIDIA_NIM_MODEL_PREFIX: Final = f"{LlmProviders.NVIDIA_NIM.value}/"
+
+
+def nvidia_nim_model_groups(deployments: Iterable[DeploymentTypedDict] | None) -> frozenset[str]:
+    return frozenset(
+        deployment["model_name"]
+        for deployment in deployments or ()
+        if deployment["litellm_params"].get("custom_llm_provider") == LlmProviders.NVIDIA_NIM.value
+        or deployment["litellm_params"].get("model", "").startswith(NVIDIA_NIM_MODEL_PREFIX)
+    )
 
 
 def nvidia_nim_router_model_in_endpoint(endpoint: str, router_models: Collection[str]) -> str | None:
diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index 529521dcba8..dfce3ab206b 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -28,7 +28,10 @@ from litellm.litellm_core_utils.url_utils import (
     validate_url,
 )
 from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
-from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_router_model_in_endpoint
+from litellm.llms.nvidia_nim.passthrough.transformation import (
+    nvidia_nim_model_groups,
+    nvidia_nim_router_model_in_endpoint,
+)
 from litellm.proxy._types import *
 from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
@@ -2052,7 +2055,7 @@ def _router_model_from_nvidia_nim_route(route: str, llm_router: Router | None) -
     if llm_router is None:
         return None
     endpoint: Final = re.sub(r"^/nvidia_nim/", "", route, flags=re.IGNORECASE)
-    return nvidia_nim_router_model_in_endpoint(endpoint, frozenset(llm_router.get_model_names()))
+    return nvidia_nim_router_model_in_endpoint(endpoint, nvidia_nim_model_groups(llm_router.get_model_list()))
 
 
 def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None:
diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index d4b1bdbdc20..65a18c1603e 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -36,7 +36,10 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.azure.passthrough.transformation import foreign_azure_deployment
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
-from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_router_model_in_endpoint
+from litellm.llms.nvidia_nim.passthrough.transformation import (
+    nvidia_nim_model_groups,
+    nvidia_nim_router_model_in_endpoint,
+)
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.passthrough.main import AsyncPassthroughStreamingResponse
 from litellm.proxy._types import *
@@ -1635,17 +1638,34 @@ async def nvidia_nim_proxy_route(
     """
     from litellm.proxy.proxy_server import llm_router
 
+    return await relay_nvidia_nim_request(
+        llm_router=llm_router,
+        endpoint=endpoint,
+        request=request,
+        request_body=await get_request_body(request),
+        user_api_key_dict=user_api_key_dict,
+    )
+
+
+async def relay_nvidia_nim_request(
+    llm_router: litellm.Router | None,
+    endpoint: str,
+    request: Request,
+    request_body: Mapping[str, object],
+    user_api_key_dict: UserAPIKeyAuth,
+) -> Response:
     model_group: Final = (
-        nvidia_nim_router_model_in_endpoint(endpoint, llm_router.get_model_names()) if llm_router else None
+        nvidia_nim_router_model_in_endpoint(endpoint, nvidia_nim_model_groups(llm_router.get_model_list()))
+        if llm_router
+        else None
     )
     if llm_router is None or model_group is None:
         rejection: Final[RelayRejection] = {
-            "error": "no LiteLLM model group in the path; call /nvidia_nim/{model_group}/v1/infer with a model "
+            "error": "no NVIDIA NIM model group in the path; call /nvidia_nim/{model_group}/v1/infer with a model "
             "from your `model_list` whose `model` starts with `nvidia_nim/`"
         }
         raise HTTPException(status_code=400, detail=rejection)
 
-    request_body: Final = await get_request_body(request)
     is_streaming_request: Final = is_passthrough_request_streaming(request_body)
     return await open_sse_before_first_byte(
         _relay_router_model(
diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py
index 81e505ea0a9..0841f82227a 100644
--- a/tests/test_litellm/proxy/auth/test_auth_utils.py
+++ b/tests/test_litellm/proxy/auth/test_auth_utils.py
@@ -844,6 +844,10 @@ def _nvidia_nim_relay_router():
                     "api_key": "k",
                 },
             },
+            {
+                "model_name": "gpt-4o",
+                "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"},
+            },
         ]
     )
 
@@ -868,6 +872,7 @@ NIM_INFER_BODY = {"input": [{"type": "image_url", "url": "data:image/png;base64,
         ("/nvidia_nim/v1/infer", NIM_INFER_BODY, None),
         ("/nvidia_nim/unknown-group/v1/infer", NIM_INFER_BODY, None),
         ("/nvidia_nim/nim-page-elements-v2/v1/infer", NIM_INFER_BODY, None),
+        ("/nvidia_nim/gpt-4o/v1/infer", NIM_INFER_BODY, None),
     ],
 )
 def test_get_model_from_request_nvidia_nim_relay_routes_use_the_model_group_in_the_path(route, request_data, expected):
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
index 4ef04a8e633..4e73a33ecd9 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
@@ -40,7 +40,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     llm_passthrough_factory_proxy_route,
     milvus_proxy_route,
     mistral_proxy_route,
-    nvidia_nim_proxy_route,
+    relay_nvidia_nim_request,
     openai_proxy_route,
     vertex_discovery_proxy_route,
     vertex_proxy_route,
@@ -5392,20 +5392,10 @@ class TestNvidiaNimProxyRoute:
         request.query_params = {}
         return request
 
-    def _install_router(self, monkeypatch, router, body: dict) -> None:
-        import litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints as ep
-        import litellm.proxy.proxy_server as proxy_server
-
-        async def fake_get_request_body(_request):
-            return dict(body)
-
-        monkeypatch.setattr(proxy_server, "llm_router", router)
-        monkeypatch.setattr(ep, "get_request_body", fake_get_request_body)
-
-    def _recording_router(self, captured: list[dict], model_names: tuple[str, ...]):
+    def _recording_router(self, captured: list[dict], deployments: dict[str, str]):
         class RecordingRouter:
-            def get_model_names(self):
-                return list(model_names)
+            def get_model_list(self):
+                return [{"model_name": name, "litellm_params": {"model": model}} for name, model in deployments.items()]
 
             async def allm_passthrough_route(self, **kwargs):
                 captured.append(kwargs)
@@ -5415,18 +5405,31 @@ class TestNvidiaNimProxyRoute:
 
         return RecordingRouter()
 
-    @pytest.mark.asyncio
-    async def test_model_group_in_the_path_selects_the_deployment_and_the_body_stays_model_free(self, monkeypatch):
-        captured: list[dict] = []
-        self._install_router(
-            monkeypatch, self._recording_router(captured, ("nim-page-elements", "nim-table")), NIM_INFER_BODY
+    async def _relay(self, llm_router, endpoint: str, body: dict, user_api_key_dict=None) -> Response:
+        return await relay_nvidia_nim_request(
+            llm_router=llm_router,
+            endpoint=endpoint,
+            request=self._request(),
+            request_body=dict(body),
+            user_api_key_dict=user_api_key_dict or UserAPIKeyAuth(api_key="hashed-token"),
         )
 
-        result = await nvidia_nim_proxy_route(
-            endpoint="nim-page-elements/v1/infer",
-            request=self._request(),
-            fastapi_response=MagicMock(spec=Response),
-            user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token", team_id="team-1"),
+    @pytest.mark.asyncio
+    async def test_model_group_in_the_path_selects_the_deployment_and_the_body_stays_model_free(self):
+        captured: list[dict] = []
+        router = self._recording_router(
+            captured,
+            {
+                "nim-page-elements": "nvidia_nim/nvidia/nemoretriever-page-elements-v2",
+                "nim-table": "nvidia_nim/nvidia/nemoretriever-table-structure-v1",
+            },
+        )
+
+        result = await self._relay(
+            router,
+            "nim-page-elements/v1/infer",
+            NIM_INFER_BODY,
+            UserAPIKeyAuth(api_key="hashed-token", team_id="team-1"),
         )
 
         (relay,) = captured
@@ -5441,65 +5444,81 @@ class TestNvidiaNimProxyRoute:
         assert result.headers["x-nim-request"] == "r1"
 
     @pytest.mark.asyncio
-    async def test_model_group_with_a_slash_is_matched_as_the_longest_leading_path(self, monkeypatch):
+    async def test_model_group_with_a_slash_is_matched_as_the_longest_leading_path(self):
         captured: list[dict] = []
-        self._install_router(
-            monkeypatch, self._recording_router(captured, ("nvidia/nemoretriever-page-elements-v2",)), NIM_INFER_BODY
+        router = self._recording_router(
+            captured, {"nvidia/nemoretriever-page-elements-v2": "nvidia_nim/nvidia/nemoretriever-page-elements-v2"}
         )
 
-        await nvidia_nim_proxy_route(
-            endpoint="nvidia/nemoretriever-page-elements-v2/v1/infer",
-            request=self._request(),
-            fastapi_response=MagicMock(spec=Response),
-            user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
-        )
+        await self._relay(router, "nvidia/nemoretriever-page-elements-v2/v1/infer", NIM_INFER_BODY)
 
         assert captured[0]["model"] == "nvidia/nemoretriever-page-elements-v2"
 
     @pytest.mark.asyncio
-    @pytest.mark.parametrize("endpoint", ["v1/infer", "unknown-group/v1/infer", "nim-page-elements-v2/v1/infer"])
-    async def test_path_without_a_configured_model_group_is_rejected_before_any_upstream_call(
-        self, monkeypatch, endpoint
-    ):
+    async def test_custom_llm_provider_marks_a_deployment_as_nim_without_the_model_prefix(self):
+        captured: list[dict] = []
+
+        class ProviderRouter:
+            def get_model_list(self):
+                return [
+                    {
+                        "model_name": "page-elements",
+                        "litellm_params": {
+                            "model": "nvidia/nemoretriever-page-elements-v2",
+                            "custom_llm_provider": "nvidia_nim",
+                        },
+                    }
+                ]
+
+            async def allm_passthrough_route(self, **kwargs):
+                captured.append(kwargs)
+                return httpx.Response(200, json={"data": []})
+
+        await self._relay(ProviderRouter(), "page-elements/v1/infer", NIM_INFER_BODY)
+
+        assert captured[0]["model"] == "page-elements"
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "endpoint",
+        ["v1/infer", "unknown-group/v1/infer", "nim-page-elements-v2/v1/infer", "gpt-4o/v1/infer"],
+    )
+    async def test_path_without_a_nim_model_group_is_rejected_before_any_upstream_call(self, endpoint):
         from fastapi import HTTPException
 
         captured: list[dict] = []
-        self._install_router(monkeypatch, self._recording_router(captured, ("nim-page-elements",)), NIM_INFER_BODY)
+        router = self._recording_router(
+            captured,
+            {"nim-page-elements": "nvidia_nim/nvidia/nemoretriever-page-elements-v2", "gpt-4o": "openai/gpt-4o"},
+        )
 
         with pytest.raises(HTTPException) as exc_info:
-            await nvidia_nim_proxy_route(
-                endpoint=endpoint,
-                request=self._request(),
-                fastapi_response=MagicMock(spec=Response),
-                user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
-            )
+            await self._relay(router, endpoint, NIM_INFER_BODY)
 
         assert exc_info.value.status_code == 400
         assert captured == []
 
     @pytest.mark.asyncio
-    async def test_no_router_is_rejected_before_any_upstream_call(self, monkeypatch):
+    async def test_no_router_is_rejected_before_any_upstream_call(self):
         from fastapi import HTTPException
 
-        self._install_router(monkeypatch, None, NIM_INFER_BODY)
-
         with pytest.raises(HTTPException) as exc_info:
-            await nvidia_nim_proxy_route(
-                endpoint="nim-page-elements/v1/infer",
-                request=self._request(),
-                fastapi_response=MagicMock(spec=Response),
-                user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
-            )
+            await self._relay(None, "nim-page-elements/v1/infer", NIM_INFER_BODY)
 
         assert exc_info.value.status_code == 400
 
     @pytest.mark.asyncio
-    async def test_upstream_rejection_is_relayed_with_its_status_body_and_headers(self, monkeypatch):
+    async def test_upstream_rejection_is_relayed_with_its_status_body_and_headers(self):
         upstream_body = {"detail": "input[0].url must be a data URL"}
 
         class RejectingRouter:
-            def get_model_names(self):
-                return ["nim-page-elements"]
+            def get_model_list(self):
+                return [
+                    {
+                        "model_name": "nim-page-elements",
+                        "litellm_params": {"model": "nvidia_nim/nvidia/nemoretriever-page-elements-v2"},
+                    }
+                ]
 
             async def allm_passthrough_route(self, **kwargs):
                 upstream_request = httpx.Request("POST", "http://nim.internal:8000/v1/infer")
@@ -5508,13 +5527,8 @@ class TestNvidiaNimProxyRoute:
                 )
                 raise httpx.HTTPStatusError("422", request=upstream_request, response=upstream)
 
-        self._install_router(monkeypatch, RejectingRouter(), {"input": [{"type": "image_url", "url": "x"}]})
-
-        result = await nvidia_nim_proxy_route(
-            endpoint="nim-page-elements/v1/infer",
-            request=self._request(),
-            fastapi_response=MagicMock(spec=Response),
-            user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token"),
+        result = await self._relay(
+            RejectingRouter(), "nim-page-elements/v1/infer", {"input": [{"type": "image_url", "url": "x"}]}
         )
 
         assert result.status_code == 422

From 082f02bd739497d6221ebe71e6885eb4a5ffeba7 Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Tue, 15 Sep 2026 22:55:29 +0000
Subject: [PATCH 53/67] refactor(model_info): drop Gemini vertex routing rule,
 keep baseline to certain capabilities

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ...odel_prices_and_context_window_backup.json | 24 ++-------------
 model_prices_and_context_window.json          | 24 ++-------------
 .../test_fallback_generalizations.py          | 30 ++-----------------
 3 files changed, 9 insertions(+), 69 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index afe2cb3deb9..3f77bd49a0d 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -58150,36 +58150,18 @@
                     "supports_reasoning": true
                 }
             },
-            {
-                "name": "vertex-gemini-ids",
-                "pattern": "^gemini-(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?:-(?:preview|exp|latest))?(?:-\\d{2}-\\d{2,4})?$",
-                "description": "A bare Gemini text-chat id at 2.5 or higher, anchored to the whole name: gemini-<major>[.minor]-(pro|flash)[-lite] with an optional preview/exp/latest tag and an optional MM-YYYY or MM-DD date. Bare Gemini ids in this map already infer vertex_ai (gemini-2.5-pro is filed under vertex_ai-language-models), so an unmapped id such as gemini-4-pro follows the same convention instead of raising 'LLM Provider NOT provided'. Anchored so a namespaced or non-chat id (gemini/..., -image, -tts) stays on its own path.",
-                "model_info": {
-                    "litellm_provider": "vertex_ai"
-                }
-            },
             {
                 "name": "gemini-family-baseline",
                 "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics))(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with a 1M context and 64K output; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with tool calling and multimodal input; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
                 "model_info": {
                     "mode": "chat",
-                    "max_input_tokens": 1048576,
-                    "max_output_tokens": 65536,
-                    "max_tokens": 65536,
                     "supports_function_calling": true,
-                    "supports_parallel_function_calling": true,
-                    "supports_vision": true,
-                    "supports_pdf_input": true,
-                    "supports_audio_input": true,
-                    "supports_video_input": true,
                     "supports_tool_choice": true,
                     "supports_system_messages": true,
+                    "supports_vision": true,
                     "supports_response_schema": true,
-                    "supports_prompt_caching": true,
-                    "supports_reasoning": true,
-                    "supports_web_search": true,
-                    "supports_url_context": true
+                    "supports_reasoning": true
                 }
             }
         ]
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index afe2cb3deb9..3f77bd49a0d 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -58150,36 +58150,18 @@
                     "supports_reasoning": true
                 }
             },
-            {
-                "name": "vertex-gemini-ids",
-                "pattern": "^gemini-(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?:-(?:preview|exp|latest))?(?:-\\d{2}-\\d{2,4})?$",
-                "description": "A bare Gemini text-chat id at 2.5 or higher, anchored to the whole name: gemini-<major>[.minor]-(pro|flash)[-lite] with an optional preview/exp/latest tag and an optional MM-YYYY or MM-DD date. Bare Gemini ids in this map already infer vertex_ai (gemini-2.5-pro is filed under vertex_ai-language-models), so an unmapped id such as gemini-4-pro follows the same convention instead of raising 'LLM Provider NOT provided'. Anchored so a namespaced or non-chat id (gemini/..., -image, -tts) stays on its own path.",
-                "model_info": {
-                    "litellm_provider": "vertex_ai"
-                }
-            },
             {
                 "name": "gemini-family-baseline",
                 "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics))(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with a 1M context and 64K output; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no pricing, so cost stays on the standard unpriced behavior rather than a guessed number.",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with tool calling and multimodal input; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
                 "model_info": {
                     "mode": "chat",
-                    "max_input_tokens": 1048576,
-                    "max_output_tokens": 65536,
-                    "max_tokens": 65536,
                     "supports_function_calling": true,
-                    "supports_parallel_function_calling": true,
-                    "supports_vision": true,
-                    "supports_pdf_input": true,
-                    "supports_audio_input": true,
-                    "supports_video_input": true,
                     "supports_tool_choice": true,
                     "supports_system_messages": true,
+                    "supports_vision": true,
                     "supports_response_schema": true,
-                    "supports_prompt_caching": true,
-                    "supports_reasoning": true,
-                    "supports_web_search": true,
-                    "supports_url_context": true
+                    "supports_reasoning": true
                 }
             }
         ]
diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
index e91a2bb4394..556a23c13be 100644
--- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
+++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
@@ -449,31 +449,6 @@ def shipped_cost_map(monkeypatch):
         set_fallback_generalizations(previous_rules)
 
 
-def test_shipped_bare_gemini_id_routes_to_vertex_ai(shipped_cost_map):
-    for model in (
-        "gemini-3.9-flash",
-        "gemini-4-pro",
-        "gemini-4-pro-preview",
-        "gemini-3.9-flash-lite-preview-09-2026",
-    ):
-        assert model not in litellm.model_cost
-        assert litellm.get_llm_provider(model=model)[1] == "vertex_ai"
-
-    assert litellm.get_llm_provider(model="gemini-2.5-pro")[1] == "vertex_ai"
-
-
-def test_shipped_gemini_routing_rule_is_anchored(shipped_cost_map):
-    for model in (
-        "gemini-4-flash-image",
-        "gemini-3.9-flash-preview-tts",
-        "gemini-2.0-flash-new",
-        "gemini-1.5-flash-new",
-        "gemini-4-flashy",
-        "gemini/gemini-4-pro",
-    ):
-        assert match_routing_generalization(model) is None, model
-
-
 @pytest.mark.parametrize(
     "model,provider",
     [
@@ -491,10 +466,11 @@ def test_shipped_gemini_baseline_resolves_unmapped_ids_provider_neutral(shipped_
     info = litellm.get_model_info(model, custom_llm_provider=provider)
     assert info["litellm_provider"] == provider
     assert info["mode"] == "chat"
-    assert info["max_input_tokens"] == 1048576
-    assert info["max_tokens"] == 65536
+    assert not info.get("max_input_tokens")
     assert info["supports_reasoning"] is True
     assert info["supports_function_calling"] is True
+    assert info["supports_tool_choice"] is True
+    assert info["supports_response_schema"] is True
     assert info["supports_vision"] is True
     assert not info.get("input_cost_per_token")
     assert not info.get("output_cost_per_token")

From 557c237b81c3e26dbbdb102d32660956e6107fa8 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 22:57:23 +0000
Subject: [PATCH 54/67] test(proxy-extras): move the LITELLM_LOG logger test
 into the CI-run proxy-extras shard

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../litellm-proxy-extras/test_litellm_proxy_extras_logging.py     | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename litellm-proxy-extras/tests/test_logging.py => tests/litellm-proxy-extras/test_litellm_proxy_extras_logging.py (100%)

diff --git a/litellm-proxy-extras/tests/test_logging.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_logging.py
similarity index 100%
rename from litellm-proxy-extras/tests/test_logging.py
rename to tests/litellm-proxy-extras/test_litellm_proxy_extras_logging.py

From c007fb992841420e38b3ae4c7221b1e4343b450a Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 23:00:08 +0000
Subject: [PATCH 55/67] fix(proxy): skip alias rewrite only for dispatched
 pass-through handlers

Match the pass-through skip to what FastAPI actually dispatched (the user-defined
endpoint marker or a provider handler's {endpoint:path} param) instead of the
mapped route prefixes, which also cover native routes such as /openai/v1/responses
and /cursor/chat/completions. Wrap the added test lines to the 120-column limit.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/auth_utils.py              |   5 +
 litellm/proxy/auth/user_api_key_auth.py       |   5 +-
 .../proxy/auth/test_user_api_key_auth.py      | 105 +++++++++++++-----
 3 files changed, 87 insertions(+), 28 deletions(-)

diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index dc304a156cf..b4c123af762 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -1967,6 +1967,11 @@ def request_dispatched_to_pass_through_endpoint(request: Request | None) -> bool
     return getattr(endpoint, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, False) is True
 
 
+def request_dispatched_to_provider_pass_through(request: Request) -> bool:
+    """Built-in provider pass-through handlers (``/anthropic/{endpoint:path}``, ...) bind ``endpoint``."""
+    return "endpoint" in request.path_params
+
+
 def get_model_from_request(
     request_data: dict,
     route: str,
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index ac0f3696680..5f3bff1e4b7 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -78,6 +78,8 @@ from litellm.proxy.auth.auth_utils import (
     iter_request_fallback_targets,
     normalize_request_route,
     pre_db_read_auth_checks,
+    request_dispatched_to_pass_through_endpoint,
+    request_dispatched_to_provider_pass_through,
     route_in_additonal_public_routes,
 )
 from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler
@@ -251,7 +253,6 @@ async def _resolve_router_settings_model_group_alias(
     """Rewrite the requested model through the key's or team's ``router_settings.model_group_alias``
     before the allowlist checks, so they authorize the model group the request is routed to.
     """
-    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers
     from litellm.proxy.proxy_server import llm_router, prisma_client, proxy_config, proxy_logging_obj
 
     if request is None or llm_router is None or not RouteChecks.is_llm_api_route(route=route):
@@ -259,7 +260,7 @@ async def _resolve_router_settings_model_group_alias(
     if request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is True:
         return
     request.scope[MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY] = True
-    if InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route):
+    if request_dispatched_to_pass_through_endpoint(request) or request_dispatched_to_provider_pass_through(request):
         return
     requested: Final = request_data.get("model")
     if not isinstance(requested, str) or await read_raw_json_body(request=request) is None:
diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
index f974e4d29d2..525f19bf7b3 100644
--- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
+++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
@@ -6,6 +6,7 @@ import subprocess
 import sys
 from contextlib import contextmanager
 from datetime import datetime, timedelta, timezone
+from functools import partial
 from pathlib import Path
 from textwrap import dedent
 from types import SimpleNamespace
@@ -8141,19 +8142,45 @@ async def test_auth_flow_enters_virtual_key_mapping_when_only_an_issuer_configur
 
 
 def _alias_router() -> litellm.Router:
-    return litellm.Router(model_list=[{"model_name": name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}} for name in ("claude-haiku", "claude-sonnet")])
+    return litellm.Router(
+        model_list=[
+            {"model_name": name, "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}}
+            for name in ("claude-haiku", "claude-sonnet")
+        ]
+    )
 
 
-def _alias_request(route: str, data: dict, content_type: str = "application/json"):
+def _alias_request(route: str, data: dict, content_type: str = "application/json", path_params: dict | None = None):
     """A request as auth sees it: the body already read once and cached alongside its parsed form."""
     from starlette.requests import Request
 
-    headers = [(b"content-type", content_type.encode())]
-    request = Request({"type": "http", "method": "POST", "path": route, "headers": headers, "query_string": b"", "parsed_body": (tuple(data), data)})
+    scope = {
+        "type": "http",
+        "method": "POST",
+        "path": route,
+        "headers": [(b"content-type", content_type.encode())],
+        "query_string": b"",
+        "path_params": path_params or {},
+        "parsed_body": (tuple(data), data),
+    }
+    request = Request(scope)
     request._body = json.dumps(data).encode()
     return request
 
 
+async def _enforce_alias_access(token: UserAPIKeyAuth, data: dict, route: str, request, router: litellm.Router):
+    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
+
+    await _enforce_key_and_fallback_model_access(
+        valid_token=token,
+        request_data=data,
+        route=route,
+        request=request,
+        llm_model_list=router.model_list,
+        llm_router=router,
+    )
+
+
 def _alias_token(monkeypatch, level: str, alias: dict, models: list) -> UserAPIKeyAuth:
     """A key whose ``router_settings.model_group_alias`` lives on the key itself or on its cached team row."""
     from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
@@ -8161,7 +8188,8 @@ def _alias_token(monkeypatch, level: str, alias: dict, models: list) -> UserAPIK
     if level == "key":
         return UserAPIKeyAuth(models=models, router_settings={"model_group_alias": alias})
     cache = UserApiKeyCache()
-    cache.set_cache(key="team_id:team-alias", value=LiteLLM_TeamTableCachedObj(team_id="team-alias", models=models, router_settings={"model_group_alias": alias}))
+    team = LiteLLM_TeamTableCachedObj(team_id="team-alias", models=models, router_settings={"model_group_alias": alias})
+    cache.set_cache(key="team_id:team-alias", value=team)
     monkeypatch.setattr(litellm.proxy.proxy_server, "user_api_key_cache", cache)
     monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", MagicMock())
     return UserAPIKeyAuth(team_id="team-alias", models=models)
@@ -8169,18 +8197,19 @@ def _alias_token(monkeypatch, level: str, alias: dict, models: list) -> UserAPIK
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("level", ["key", "team"])
-@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/messages", "/v1/embeddings"])
+@pytest.mark.parametrize(
+    "route",
+    ["/v1/chat/completions", "/v1/messages", "/v1/embeddings", "/openai/v1/responses", "/cursor/chat/completions"],
+)
 async def test_router_settings_model_group_alias_authorizes_target_for_key(monkeypatch, level, route):
     """LIT-3054: a key allowed only the alias target must be able to call the alias, and a key not
     allowed the target must still be denied even when the alias itself is what it requested."""
-    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
-
     router = _alias_router()
     monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
     data = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]}
     request = _alias_request(route, data)
     token = _alias_token(monkeypatch, level, {"AgentX-LLM": "claude-haiku"}, ["claude-haiku"])
-    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route=route, request=request, llm_model_list=router.model_list, llm_router=router)
+    await _enforce_alias_access(token, data, route, request, router)
     assert data["model"] == "claude-haiku"
     assert (await request.json())["model"] == "claude-haiku"
     assert json.loads(await request.body())["model"] == "claude-haiku"
@@ -8190,21 +8219,20 @@ async def test_router_settings_model_group_alias_authorizes_target_for_key(monke
     denied = _alias_token(monkeypatch, level, {"AgentX-LLM": "claude-sonnet"}, ["claude-haiku"])
     denied_data = {"model": "AgentX-LLM"}
     with pytest.raises(ProxyException) as exc:
-        await _enforce_key_and_fallback_model_access(valid_token=denied, request_data=denied_data, route=route, request=_alias_request(route, denied_data), llm_model_list=router.model_list, llm_router=router)
+        await _enforce_alias_access(denied, denied_data, route, _alias_request(route, denied_data), router)
     assert "claude-sonnet" in exc.value.message
 
 
 @pytest.mark.asyncio
 async def test_router_settings_model_group_alias_leaves_form_bodies_alone(monkeypatch):
     """LIT-3054: a multipart body cannot be re-serialized as JSON, so auth must not rewrite it."""
-    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
-
     router = _alias_router()
     monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
     data = {"model": "AgentX-LLM"}
-    request = _alias_request("/v1/audio/transcriptions", data, content_type="multipart/form-data; boundary=x")
+    route = "/v1/audio/transcriptions"
+    request = _alias_request(route, data, content_type="multipart/form-data; boundary=x")
     token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku", "AgentX-LLM"])
-    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route="/v1/audio/transcriptions", request=request, llm_model_list=router.model_list, llm_router=router)
+    await _enforce_alias_access(token, data, route, request, router)
     assert data["model"] == "AgentX-LLM"
     assert get_client_requested_model(request) is None
 
@@ -8212,7 +8240,6 @@ async def test_router_settings_model_group_alias_leaves_form_bodies_alone(monkey
 @pytest.mark.asyncio
 async def test_router_settings_model_group_alias_rewrite_keeps_query_params_out_of_body(monkeypatch):
     """LIT-3054: auth merges query params into its own copy of the body; the rewrite must not forward them."""
-    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
     from litellm.proxy.common_utils.http_parsing_utils import _read_request_body, populate_request_with_path_params
 
     router = _alias_router()
@@ -8223,25 +8250,42 @@ async def test_router_settings_model_group_alias_rewrite_keeps_query_params_out_
     data = populate_request_with_path_params(request_data=await _read_request_body(request), request=request)
     assert data["api-version"] == "2024-10-21"
     token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku"])
-    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route="/v1/chat/completions", request=request, llm_model_list=router.model_list, llm_router=router)
+    await _enforce_alias_access(token, data, "/v1/chat/completions", request, router)
     downstream = await _read_request_body(request)
     assert downstream == {**body, "model": "claude-haiku"}
     assert json.loads(await request.body()) == downstream
     assert await request.json() == downstream
 
 
-@pytest.mark.asyncio
-async def test_router_settings_model_group_alias_leaves_pass_through_bodies_alone(monkeypatch):
-    """LIT-3054: pass-through routes forward the body verbatim to the provider, so auth must not rewrite it."""
-    from litellm.proxy.auth.user_api_key_auth import _enforce_key_and_fallback_model_access
+def _user_defined_pass_through_endpoint():
+    from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER
 
+    async def endpoint():
+        return None
+
+    setattr(endpoint, LITELLM_PASS_THROUGH_ENDPOINT_MARKER, True)
+    return endpoint
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("user_defined", [False, True])
+async def test_router_settings_model_group_alias_leaves_pass_through_bodies_alone(monkeypatch, user_defined):
+    """LIT-3054: pass-through handlers forward the body verbatim to the provider, so auth must not rewrite it.
+    Built-in provider handlers bind ``{endpoint:path}``; user-defined ones carry the pass-through marker."""
     router = _alias_router()
     monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router)
     data = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]}
-    route = "/anthropic/v1/messages"
-    request = _alias_request(route, data)
+    route = "/custom-upstream/chat" if user_defined else "/anthropic/v1/messages"
+    request = _alias_request(route, data, path_params={} if user_defined else {"endpoint": "v1/messages"})
+    if user_defined:
+        request.scope["endpoint"] = _user_defined_pass_through_endpoint()
+        LiteLLMRoutes.openai_routes.value.append(route)
     token = _alias_token(monkeypatch, "key", {"AgentX-LLM": "claude-haiku"}, ["claude-haiku", "AgentX-LLM"])
-    await _enforce_key_and_fallback_model_access(valid_token=token, request_data=data, route=route, request=request, llm_model_list=router.model_list, llm_router=router)
+    try:
+        await _enforce_alias_access(token, data, route, request, router)
+    finally:
+        if user_defined:
+            LiteLLMRoutes.openai_routes.value.remove(route)
     assert data["model"] == "AgentX-LLM"
     assert (await request.json())["model"] == "AgentX-LLM"
     assert get_client_requested_model(request) is None
@@ -8262,13 +8306,22 @@ async def test_router_settings_model_group_alias_authorizes_target_for_team(monk
     token = _alias_token(monkeypatch, "team", {"AgentX-LLM": target}, ["claude-haiku"])
     token.team_models = ["claude-haiku"]
     data = {"model": "AgentX-LLM", "messages": [{"role": "user", "content": "hi"}]}
-    request = _alias_request("/v1/chat/completions", data)
+    route = "/v1/chat/completions"
+    request = _alias_request(route, data)
+    authorize = partial(
+        _authorize_authenticated_request,
+        user_api_key_auth_obj=token,
+        request=request,
+        request_data=data,
+        route=route,
+        api_key="sk-test",
+    )
     if expect_denied:
         with pytest.raises(ProxyException) as exc:
-            await _authorize_authenticated_request(user_api_key_auth_obj=token, request=request, request_data=data, route="/v1/chat/completions", api_key="sk-test")
+            await authorize()
         assert exc.value.type == ProxyErrorTypes.team_model_access_denied
         assert target in exc.value.message
         return
-    await _authorize_authenticated_request(user_api_key_auth_obj=token, request=request, request_data=data, route="/v1/chat/completions", api_key="sk-test")
+    await authorize()
     assert (await request.json())["model"] == target
     assert get_client_requested_model(request) == "AgentX-LLM"

From 0c65dcff22948008b3a8e59078aaf8c084e10e54 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 23:15:34 +0000
Subject: [PATCH 56/67] fix(proxy): strip line breaks from alias resolution
 debug log

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/user_api_key_auth.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 5f3bff1e4b7..15dd58baf35 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -273,7 +273,11 @@ async def _resolve_router_settings_model_group_alias(
     target: Final = resolve_model_group_alias(settings.get("model_group_alias"), requested)
     if target is None or target == requested:
         return
-    verbose_proxy_logger.debug("router_settings.model_group_alias resolved %s -> %s before auth", requested, target)
+    verbose_proxy_logger.debug(
+        "router_settings.model_group_alias resolved %s -> %s before auth",
+        requested.replace("\r", "").replace("\n", ""),
+        target.replace("\r", "").replace("\n", ""),
+    )
     request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested)
     rewrite_request_model(request_data, request, target)
 

From 726430c0e7e812804b39251aa11c8df569c731d7 Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Tue, 15 Sep 2026 23:16:32 +0000
Subject: [PATCH 57/67] refactor(model_info): scope Gemini baseline to
 first-party chat ids

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ...odel_prices_and_context_window_backup.json | 11 ++--
 model_prices_and_context_window.json          | 11 ++--
 .../test_fallback_generalizations.py          | 53 +++++++++++++------
 .../test_get_supported_openai_params.py       | 50 +++++------------
 4 files changed, 62 insertions(+), 63 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 3f77bd49a0d..9569f29aa0e 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -58151,17 +58151,20 @@
                 }
             },
             {
-                "name": "gemini-family-baseline",
-                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics))(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with tool calling and multimodal input; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
+                "name": "gemini-chat-baseline",
+                "pattern": "^(?:gemini|vertex_ai)/gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2\\.[5-9]|[3-9](?:\\.\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "A first-party Gemini text-chat id at 2.5 or higher, as gemini/<id> or vertex_ai/<id> (get_model_info tries the provider-prefixed name first, so a bare id under those providers matches too): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. Scoped to Google's own namespaces because resellers wrap the same models differently (one exposes them as mode responses without reasoning), so nothing below is certain there. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. 2.5 is the floor: every first-party text-chat Gemini from 2.5 onward in this map carries all of these fields, 1.5 and 2.0 do not and are fully mapped. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
                 "model_info": {
                     "mode": "chat",
+                    "supports_reasoning": true,
                     "supports_function_calling": true,
                     "supports_tool_choice": true,
                     "supports_system_messages": true,
                     "supports_vision": true,
                     "supports_response_schema": true,
-                    "supports_reasoning": true
+                    "supports_pdf_input": true,
+                    "supports_prompt_caching": true,
+                    "supports_web_search": true
                 }
             }
         ]
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 3f77bd49a0d..9569f29aa0e 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -58151,17 +58151,20 @@
                 }
             },
             {
-                "name": "gemini-family-baseline",
-                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics))(?:2\\.[5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "Any Gemini text-chat id at 2.5 or higher under any provider namespace (bare, gemini/, vertex_ai/, databricks-gemini-3-1-pro, ...): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The lookahead excludes the tts, image, live, audio, embedding, computer-use and robotics lines, which are not text-chat models and carry different limits and modes. 2.5 is the floor because every Gemini from 2.5 onward is a thinking model with tool calling and multimodal input; 1.5 and 2.0 are not and are fully mapped. Carries the model-family facts every such Gemini shares, so an unmapped gemini-4-pro keeps reasoning params and tool calling instead of having them dropped; it carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
+                "name": "gemini-chat-baseline",
+                "pattern": "^(?:gemini|vertex_ai)/gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2\\.[5-9]|[3-9](?:\\.\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "A first-party Gemini text-chat id at 2.5 or higher, as gemini/<id> or vertex_ai/<id> (get_model_info tries the provider-prefixed name first, so a bare id under those providers matches too): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. Scoped to Google's own namespaces because resellers wrap the same models differently (one exposes them as mode responses without reasoning), so nothing below is certain there. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. 2.5 is the floor: every first-party text-chat Gemini from 2.5 onward in this map carries all of these fields, 1.5 and 2.0 do not and are fully mapped. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
                 "model_info": {
                     "mode": "chat",
+                    "supports_reasoning": true,
                     "supports_function_calling": true,
                     "supports_tool_choice": true,
                     "supports_system_messages": true,
                     "supports_vision": true,
                     "supports_response_schema": true,
-                    "supports_reasoning": true
+                    "supports_pdf_input": true,
+                    "supports_prompt_caching": true,
+                    "supports_web_search": true
                 }
             }
         ]
diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
index 556a23c13be..a4fbd79187e 100644
--- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
+++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
@@ -453,45 +453,64 @@ def shipped_cost_map(monkeypatch):
     "model,provider",
     [
         ("gemini-4-pro", "gemini"),
-        ("gemini-3.9-flash", "vertex_ai"),
-        ("databricks-gemini-4-1-pro", "databricks"),
+        ("gemini/gemini-4-pro", None),
+        ("gemini-3.9-flash-lite-preview-09-2026", "vertex_ai"),
+        ("vertex_ai/gemini-4-pro", None),
         ("gemini-4-pro-preview-customtools", "gemini"),
     ],
 )
-def test_shipped_gemini_baseline_resolves_unmapped_ids_provider_neutral(shipped_cost_map, model, provider):
+def test_shipped_gemini_chat_baseline_resolves_first_party_unmapped_ids(shipped_cost_map, model, provider):
     assert model not in litellm.model_cost
     if provider == "gemini":
         assert f"gemini/{model}" not in litellm.model_cost
 
     info = litellm.get_model_info(model, custom_llm_provider=provider)
-    assert info["litellm_provider"] == provider
+    assert info["litellm_provider"] == (provider or model.split("/")[0])
     assert info["mode"] == "chat"
     assert not info.get("max_input_tokens")
     assert info["supports_reasoning"] is True
     assert info["supports_function_calling"] is True
     assert info["supports_tool_choice"] is True
-    assert info["supports_response_schema"] is True
+    assert info["supports_system_messages"] is True
     assert info["supports_vision"] is True
+    assert info["supports_response_schema"] is True
+    assert info["supports_pdf_input"] is True
+    assert info["supports_prompt_caching"] is True
+    assert info["supports_web_search"] is True
     assert not info.get("input_cost_per_token")
     assert not info.get("output_cost_per_token")
 
 
-def test_shipped_gemini_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cost_map):
+def test_shipped_gemini_chat_baseline_skips_reseller_namespaces(shipped_cost_map):
+    for model, provider in (
+        ("google/gemini-4-pro", "perplexity"),
+        ("google/gemini-4-pro", "openrouter"),
+        ("databricks-gemini-4-1-pro", "databricks"),
+    ):
+        with pytest.raises(Exception, match="isn't mapped yet"):
+            litellm.get_model_info(model, custom_llm_provider=provider)
+
+    assert match_capability_generalizations("google/gemini-4-pro") is None
+
+
+def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cost_map):
     for model in (
-        "gemini-4-flash-image",
-        "gemini-3.9-flash-preview-tts",
-        "gemini-4-flash-live-preview",
-        "gemini-4-flash-native-audio",
-        "gemini-embedding-4",
-        "gemini-2.5-computer-use-preview-12-2026",
-        "gemini-2.0-flash-new",
-        "gemini-1.5-pro-new",
-        "gemini-4-flashy",
+        "gemini/gemini-4-flash-image",
+        "gemini/gemini-3.9-flash-preview-tts",
+        "gemini/gemini-4-flash-live-preview",
+        "gemini/gemini-4-flash-native-audio",
+        "gemini/gemini-embedding-4",
+        "gemini/gemini-2.5-computer-use-preview-12-2026",
+        "gemini/gemini-2.0-flash-new",
+        "gemini/gemini-1.5-pro-new",
+        "gemini/gemini-4-flashy",
+        "gemini/gemini-4-flash-transcribe",
+        "gemini/gemini-4-flash-live-translate-preview",
     ):
         assert match_capability_generalizations(model) is None, model
 
 
-def test_shipped_gemini_baseline_keeps_reasoning_effort_on_unmapped_model(shipped_cost_map):
+def test_shipped_gemini_chat_baseline_keeps_reasoning_effort_on_unmapped_model(shipped_cost_map):
     assert litellm.supports_reasoning(model="gemini-4-pro", custom_llm_provider="gemini") is True
 
     optional_params = litellm.utils.get_optional_params(
@@ -505,7 +524,7 @@ def test_shipped_gemini_baseline_keeps_reasoning_effort_on_unmapped_model(shippe
     assert optional_params["thinkingConfig"]["includeThoughts"] is True
 
 
-def test_shipped_gemini_baseline_loses_to_exact_entries(shipped_cost_map):
+def test_shipped_gemini_chat_baseline_loses_to_exact_entries(shipped_cost_map):
     model = "gemini-2.5-flash-lite"
     info = litellm.get_model_info(model, custom_llm_provider="gemini")
     entry = litellm.model_cost["gemini/gemini-2.5-flash-lite"]
diff --git a/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py b/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py
index 722818598af..f9e285cf9fb 100644
--- a/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py
+++ b/tests/test_litellm/litellm_core_utils/test_get_supported_openai_params.py
@@ -1,7 +1,5 @@
-
 import pytest
 
-
 from litellm.litellm_core_utils.get_supported_openai_params import (
     get_supported_openai_params,
 )
@@ -33,9 +31,7 @@ def test_base_model_label_alone_lacks_bedrock_tools():
     """The label by itself does not advertise tools; this is what made the union
     necessary. Guards against the discrepancy disappearing (and the regression test
     above silently passing for the wrong reason)."""
-    params = get_supported_openai_params(
-        model=BEDROCK_LABEL, custom_llm_provider="bedrock"
-    )
+    params = get_supported_openai_params(model=BEDROCK_LABEL, custom_llm_provider="bedrock")
 
     assert params is not None
     assert "tools" not in params
@@ -46,14 +42,8 @@ def test_base_model_is_additive_not_replacement():
 
     Bedrock: real id supports ``tools`` but not the label's reasoning hint; the union
     must contain the real model's ``tools`` regardless of the label being a subset."""
-    real_only = set(
-        get_supported_openai_params(
-            model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"
-        )
-    )
-    label_only = set(
-        get_supported_openai_params(model=BEDROCK_LABEL, custom_llm_provider="bedrock")
-    )
+    real_only = set(get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"))
+    label_only = set(get_supported_openai_params(model=BEDROCK_LABEL, custom_llm_provider="bedrock"))
     combined = set(
         get_supported_openai_params(
             model=BEDROCK_REAL_MODEL,
@@ -70,19 +60,15 @@ def test_base_model_is_additive_not_replacement():
 def test_base_model_adds_capabilities_the_real_model_lacks():
     """Regression for #27717 (the behavior the union must preserve).
 
-    ``gemini-3.1-pro`` isn't in the cost map so it advertises no reasoning support,
+    ``gemini-exp-9999`` isn't in the cost map so it advertises no reasoning support,
     but the registered ``gemini-3.1-pro-preview`` base_model does. The hint must add
     ``reasoning_effort``/``thinking`` without the call erroring."""
-    real_only = set(
-        get_supported_openai_params(
-            model="gemini-3.1-pro", custom_llm_provider="gemini"
-        )
-    )
+    real_only = set(get_supported_openai_params(model="gemini-exp-9999", custom_llm_provider="gemini"))
     assert "reasoning_effort" not in real_only
 
     combined = set(
         get_supported_openai_params(
-            model="gemini-3.1-pro",
+            model="gemini-exp-9999",
             custom_llm_provider="gemini",
             base_model="gemini-3.1-pro-preview",
         )
@@ -93,21 +79,15 @@ def test_base_model_adds_capabilities_the_real_model_lacks():
 
 def test_no_base_model_is_unchanged():
     """Omitting ``base_model`` must resolve purely from ``model``."""
-    with_none = get_supported_openai_params(
-        model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock", base_model=None
-    )
-    plain = get_supported_openai_params(
-        model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"
-    )
+    with_none = get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock", base_model=None)
+    plain = get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock")
 
     assert with_none == plain
 
 
 def test_base_model_equal_to_model_is_unchanged():
     """A ``base_model`` identical to ``model`` must not double-resolve or reorder."""
-    plain = get_supported_openai_params(
-        model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock"
-    )
+    plain = get_supported_openai_params(model=BEDROCK_REAL_MODEL, custom_llm_provider="bedrock")
     same = get_supported_openai_params(
         model=BEDROCK_REAL_MODEL,
         custom_llm_provider="bedrock",
@@ -152,14 +132,10 @@ def test_bedrock_converse_alias_resolves_like_bedrock():
     params saw no Bedrock capabilities for a Converse model invoked via the alias."""
     anthropic_model = "bedrock/converse/us.anthropic.claude-sonnet-4-6"
 
-    via_alias = get_supported_openai_params(
-        model=anthropic_model, custom_llm_provider="bedrock_converse"
-    )
+    via_alias = get_supported_openai_params(model=anthropic_model, custom_llm_provider="bedrock_converse")
 
     assert via_alias is not None
-    assert via_alias == get_supported_openai_params(
-        model=anthropic_model, custom_llm_provider="bedrock"
-    )
+    assert via_alias == get_supported_openai_params(model=anthropic_model, custom_llm_provider="bedrock")
     assert "web_search_options" not in via_alias
     assert "tools" in via_alias
 
@@ -167,9 +143,7 @@ def test_bedrock_converse_alias_resolves_like_bedrock():
 def test_bedrock_converse_alias_keeps_nova_web_search_options():
     """Nova on the ``bedrock_converse`` alias still advertises web_search_options, proving the
     alias routes through the model-aware config rather than a blanket Bedrock default."""
-    nova_params = get_supported_openai_params(
-        model="amazon.nova-pro-v1:0", custom_llm_provider="bedrock_converse"
-    )
+    nova_params = get_supported_openai_params(model="amazon.nova-pro-v1:0", custom_llm_provider="bedrock_converse")
 
     assert nova_params is not None
     assert "web_search_options" in nova_params

From 69c32122200131efe14ac5ca4b70e3609e46b910 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 15 Sep 2026 16:21:37 -0700
Subject: [PATCH 58/67] fix(ui): gate the usage export on range coverage, not
 on a fetch being in flight

A loading flag only flips once the fetch effect runs, so the render right after a
date or filter change still reported the previous range as loaded and let an export
read its rows. Stamp the completed range on the hook and compare it during render
instead, the way the tiles already do.

Also stop the failure banner claiming a page loaded when the first request is what
failed, which left it reading 1/1.
---
 .../components/EntityUsage/EntityUsage.tsx    |  4 +-
 .../_components/components/UsagePageView.tsx  |  5 +-
 .../hooks/usePaginatedDailyActivity.test.ts   | 95 +++++++++++++++++++
 .../hooks/usePaginatedDailyActivity.ts        | 16 +++-
 .../EntityUsageExport/UsageExportHeader.tsx   |  1 -
 .../exportBlockedReason.test.ts               | 17 ++--
 .../EntityUsageExport/exportBlockedReason.ts  | 14 +--
 .../shared/PaginationStatusAlerts.test.tsx    | 17 +++-
 .../shared/PaginationStatusAlerts.tsx         | 10 +-
 9 files changed, 144 insertions(+), 35 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx
index debc5602277..6687bd4df03 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx
@@ -146,11 +146,11 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
 
   const {
     data: spendDataRaw,
-    loading,
     isFetchingMore,
     progress,
     cancelled,
     failed,
+    coversRange,
     cancel,
   } = usePaginatedDailyActivity({
     fetchFn,
@@ -664,7 +664,7 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
     { key: "endpoints", label: "Endpoint Activity", content: <EndpointUsage userSpendData={spendData} /> },
   ];
 
-  const spendFetchState = { loading, isFetchingMore, cancelled, failed };
+  const spendFetchState = { coversRange, cancelled, failed };
 
   return (
     <div style={{ width: "100%" }} className="relative">
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
index 31c9288c911..691c5dc839a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
@@ -250,9 +250,10 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
 
   const loading = aggregatedLoading || paginatedResult.loading;
 
+  // Read through the same range stamp as the tiles, so the export is blocked from the first
+  // render of a new range rather than from whenever the fetch effect gets around to running.
   const spendFetchState = {
-    loading,
-    isFetchingMore: paginatedResult.isFetchingMore,
+    coversRange: activeAggregated !== null || paginatedResult.coversRange,
     cancelled: paginatedResult.cancelled,
     failed: paginatedResult.failed,
   };
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts
index 5257e5fc8a6..aa6cf64483a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.test.ts
@@ -180,6 +180,20 @@ describe("usePaginatedDailyActivity failure reporting", () => {
     consoleError.mockRestore();
   });
 
+  it("reports no pages loaded when the very first request is what failed", async () => {
+    const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
+    const fetchFn = vi.fn(() => Promise.reject(new Error("page 1 never came back")));
+
+    const { result } = renderHook(() =>
+      usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
+    );
+
+    await waitFor(() => expect(result.current.failed).toBe(true), { timeout: 5000 });
+
+    expect(result.current.progress).toEqual({ currentPage: 0, totalPages: 0 });
+    consoleError.mockRestore();
+  });
+
   it("stays unfailed when every page arrives", async () => {
     const pages = [
       firstPage,
@@ -222,3 +236,84 @@ describe("usePaginatedDailyActivity failure reporting", () => {
     consoleError.mockRestore();
   });
 });
+
+describe("usePaginatedDailyActivity range coverage", () => {
+  const start = new Date("2026-08-10");
+  const end = new Date("2026-08-17");
+  const singlePage = { results: [dayOf("2026-08-16", 2)], metadata: { total_pages: 1, page: 1, total_spend: 2 } };
+
+  it("does not cover the range while the hook is disabled", () => {
+    const fetchFn = vi.fn(() => Promise.resolve(singlePage));
+
+    const { result } = renderHook(() =>
+      usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: false }),
+    );
+
+    expect(result.current.coversRange).toBe(false);
+    expect(fetchFn).not.toHaveBeenCalled();
+  });
+
+  it("covers the range only once every page of it has landed", async () => {
+    const pages = [
+      { results: [dayOf("2026-08-16", 2)], metadata: { total_pages: 2, page: 1, total_spend: 2 } },
+      { results: [dayOf("2026-08-15", 1)], metadata: { total_pages: 2, page: 2, total_spend: 1 } },
+    ];
+    const fetchFn = vi.fn((_token: string, _start: Date, _end: Date, page: number) => Promise.resolve(pages[page - 1]));
+
+    const { result } = renderHook(() =>
+      usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled: true }),
+    );
+
+    expect(result.current.coversRange).toBe(false);
+    await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
+  });
+
+  it("never reports a range as covered while the data on screen is empty", async () => {
+    // Disabling the hook empties the data. Re-enabling it asks for the same args the last
+    // completed fetch used, so coverage that survives the disable would vouch for nothing.
+    const seen: Array<{ coversRange: boolean; rows: number }> = [];
+    const fetchFn = vi.fn(() => Promise.resolve(singlePage));
+
+    const { result, rerender } = renderHook(
+      ({ enabled }: { enabled: boolean }) => {
+        const activity = usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, null], enabled });
+        seen.push({ coversRange: activity.coversRange, rows: activity.data.results.length });
+        return activity;
+      },
+      { initialProps: { enabled: true } },
+    );
+
+    await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
+
+    rerender({ enabled: false });
+    rerender({ enabled: true });
+
+    await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
+    expect(seen.filter((render) => render.coversRange && render.rows === 0)).toEqual([]);
+  });
+
+  it("stops covering the range on the very render the args change, not once an effect catches up", async () => {
+    // The render after a filter change still holds the previous filter's rows, so resetting
+    // coverage inside the fetch effect would leave a paint where the export reads them as the
+    // new range. That paint is the whole thing the gate exists to stop.
+    const seen: Array<{ filter: string; coversRange: boolean }> = [];
+    const fetchFn = vi.fn(() => Promise.resolve(singlePage));
+
+    const { result, rerender } = renderHook(
+      ({ filter }: { filter: string }) => {
+        const activity = usePaginatedDailyActivity({ fetchFn, args: ["tok", start, end, filter], enabled: true });
+        seen.push({ filter, coversRange: activity.coversRange });
+        return activity;
+      },
+      { initialProps: { filter: "team-a" } },
+    );
+
+    await waitFor(() => expect(result.current.coversRange).toBe(true), { timeout: 5000 });
+
+    rerender({ filter: "team-b" });
+
+    const rendersForNewFilter = seen.filter((render) => render.filter === "team-b");
+    expect(rendersForNewFilter.length).toBeGreaterThan(0);
+    expect(rendersForNewFilter.map((render) => render.coversRange)).not.toContain(true);
+  });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts
index aef0021cc3d..0c1d79cb112 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity.ts
@@ -61,8 +61,8 @@ interface UsePaginatedDailyActivityReturn {
   isFetchingMore: boolean;
   progress: PaginationProgress;
   cancelled: boolean;
-  /** A page request threw, so `data` covers only part of the requested range. */
   failed: boolean;
+  coversRange: boolean;
   cancel: () => void;
 }
 
@@ -203,6 +203,7 @@ export function usePaginatedDailyActivity({
   });
   const [cancelled, setCancelled] = useState(false);
   const [failed, setFailed] = useState(false);
+  const [completedKey, setCompletedKey] = useState<string | null>(null);
 
   const fetchIdRef = useRef(0);
   const cancelledRef = useRef(false);
@@ -216,6 +217,11 @@ export function usePaginatedDailyActivity({
   // Stable serialised key so the effect only re-runs when the arg *values* change.
   const argsKey = JSON.stringify(args);
 
+  // Stamped like the data itself and compared during render, so the render that follows an arg
+  // change already reports the new range as uncovered. Clearing it inside the fetch effect would
+  // be one render too late, leaving a paint where an export reads the previous range's rows.
+  const coversRange = enabled && completedKey === argsKey;
+
   const cancel = useCallback(() => {
     cancelledRef.current = true;
     setCancelled(true);
@@ -234,6 +240,7 @@ export function usePaginatedDailyActivity({
       setProgress({ currentPage: 0, totalPages: 0 });
       setCancelled(false);
       setFailed(false);
+      setCompletedKey(null);
       return;
     }
 
@@ -257,7 +264,7 @@ export function usePaginatedDailyActivity({
       const currentArgs = argsRef.current;
       setLoading(true);
       setIsFetchingMore(false);
-      setProgress({ currentPage: 1, totalPages: 1 });
+      setProgress({ currentPage: 0, totalPages: 0 });
 
       if (aggregatedFetchFn) {
         try {
@@ -266,6 +273,7 @@ export function usePaginatedDailyActivity({
           setData(aggregated);
           setProgress({ currentPage: 1, totalPages: 1 });
           setLoading(false);
+          setCompletedKey(argsKey);
           return;
         } catch (error) {
           if (isStale()) return;
@@ -288,6 +296,7 @@ export function usePaginatedDailyActivity({
 
         if (totalPages <= 1) {
           setLoading(false);
+          setCompletedKey(argsKey);
           return;
         }
 
@@ -333,6 +342,7 @@ export function usePaginatedDailyActivity({
         }
 
         setIsFetchingMore(false);
+        setCompletedKey(argsKey);
       } catch (error) {
         if (!isStale()) {
           console.error("Error fetching daily activity:", error);
@@ -356,5 +366,5 @@ export function usePaginatedDailyActivity({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [enabled, fetchFn, aggregatedFetchFn, argsKey]);
 
-  return { data, loading, isFetchingMore, progress, cancelled, failed, cancel };
+  return { data, loading, isFetchingMore, progress, cancelled, failed, coversRange, cancel };
 }
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx
index 3c4c7f6f4ee..388a211d9bb 100644
--- a/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx
+++ b/ui/litellm-dashboard/src/components/EntityUsageExport/UsageExportHeader.tsx
@@ -34,7 +34,6 @@ interface UsageExportHeaderProps {
   customTitle?: string;
   compactLayout?: boolean;
   teams?: Team[];
-  /** Set to block the export and explain why; see getExportBlockedReason. */
   exportBlockedReason?: string;
 }
 
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts
index 0b32aef52fa..e39b01a5dea 100644
--- a/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts
+++ b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.test.ts
@@ -3,35 +3,30 @@ import { describe, expect, it } from "vitest";
 import { getExportBlockedReason, type UsageFetchState } from "./exportBlockedReason";
 
 const state = (overrides: Partial<UsageFetchState> = {}): UsageFetchState => ({
-  loading: false,
-  isFetchingMore: false,
+  coversRange: true,
   cancelled: false,
   failed: false,
   ...overrides,
 });
 
 describe("getExportBlockedReason", () => {
-  it("lets the export through once the range has fully loaded", () => {
+  it("lets the export through once the data on screen covers the range", () => {
     expect(getExportBlockedReason(state())).toBeUndefined();
   });
 
-  it("blocks the first load, before any page has arrived", () => {
-    expect(getExportBlockedReason(state({ loading: true }))).toMatch(/still loading/i);
-  });
-
-  it("blocks while later pages are still arriving, which is when a CSV silently under-reports", () => {
-    expect(getExportBlockedReason(state({ isFetchingMore: true }))).toMatch(/still loading/i);
+  it("blocks whenever the data on screen does not cover the range, which is when a CSV silently under-reports", () => {
+    expect(getExportBlockedReason(state({ coversRange: false }))).toMatch(/still loading/i);
   });
 
   it("blocks after a stopped fetch and says a reload is what fixes it", () => {
-    const reason = getExportBlockedReason(state({ cancelled: true }));
+    const reason = getExportBlockedReason(state({ coversRange: false, cancelled: true }));
 
     expect(reason).toMatch(/stopped/i);
     expect(reason).toMatch(/reload/i);
   });
 
   it("blocks after a failed page and names the failure rather than the stop", () => {
-    const reason = getExportBlockedReason(state({ failed: true, cancelled: true }));
+    const reason = getExportBlockedReason(state({ coversRange: false, failed: true, cancelled: true }));
 
     expect(reason).toMatch(/failed to load/i);
     expect(reason).not.toMatch(/stopped/i);
diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts
index ebe67f1ccfe..71408ba8f3f 100644
--- a/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts
+++ b/ui/litellm-dashboard/src/components/EntityUsageExport/exportBlockedReason.ts
@@ -1,21 +1,13 @@
 export interface UsageFetchState {
-  loading: boolean;
-  isFetchingMore: boolean;
+  coversRange: boolean;
   cancelled: boolean;
   failed: boolean;
 }
 
-/** Why exporting what is on screen would under-report, or undefined once it covers the whole range. */
-export const getExportBlockedReason = ({
-  loading,
-  isFetchingMore,
-  cancelled,
-  failed,
-}: UsageFetchState): string | undefined => {
+export const getExportBlockedReason = ({ coversRange, cancelled, failed }: UsageFetchState): string | undefined => {
   if (failed) return "Some spend data failed to load, so an export would under-report. Reload the page to try again.";
   if (cancelled)
     return "Loading was stopped before the whole range arrived, so an export would under-report. Reload the page to load it all.";
-  if (loading || isFetchingMore)
-    return "Spend data is still loading, so an export would under-report. Wait for it to finish.";
+  if (!coversRange) return "Spend data is still loading, so an export would under-report. Wait for it to finish.";
   return undefined;
 };
diff --git a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx
index 1eefc43ecb4..93d464b6615 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.test.tsx
@@ -45,10 +45,25 @@ describe("PaginationStatusAlerts", () => {
     );
 
     expect(
-      screen.getByText(/Fetching spend data failed, so the totals below cover only part of the range \(7\/42 pages/),
+      screen.getByText(/Fetching spend data failed, so the totals below cover only 7 of 42 pages of the range/),
     ).toBeInTheDocument();
   });
 
+  it("does not claim a page loaded when the very first request is what failed", () => {
+    render(
+      <PaginationStatusAlerts
+        isFetchingMore={false}
+        cancelled={false}
+        failed={true}
+        progress={{ currentPage: 0, totalPages: 0 }}
+        cancel={vi.fn()}
+      />,
+    );
+
+    expect(screen.getByText(/failed before any of it arrived/)).toBeInTheDocument();
+    expect(screen.queryByText(/pages of the range/)).not.toBeInTheDocument();
+  });
+
   it("shows only the failure when a stopped fetch also failed", () => {
     render(
       <PaginationStatusAlerts
diff --git a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx
index ac06cfd46cf..e9357993dd0 100644
--- a/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx
+++ b/ui/litellm-dashboard/src/components/shared/PaginationStatusAlerts.tsx
@@ -12,6 +12,11 @@ interface PaginationStatusAlertsProps {
   failed?: boolean;
 }
 
+const failureMessage = (subject: string, progress: { currentPage: number; totalPages: number }) =>
+  progress.currentPage === 0
+    ? `Fetching ${subject} failed before any of it arrived, so the totals below are empty rather than final. Reload the page to try again.`
+    : `Fetching ${subject} failed, so the totals below cover only ${progress.currentPage} of ${progress.totalPages} pages of the range. Reload the page to try again.`;
+
 const PaginationStatusAlerts = ({
   isFetchingMore,
   cancelled,
@@ -42,10 +47,7 @@ const PaginationStatusAlerts = ({
     )}
     {failed && (
       <Alert variant="error" className="mb-2">
-        <AlertDescription className="text-inherit">
-          Fetching {subject} failed, so the totals below cover only part of the range ({progress.currentPage}/
-          {progress.totalPages} pages loaded). Reload the page to try again.
-        </AlertDescription>
+        <AlertDescription className="text-inherit">{failureMessage(subject, progress)}</AlertDescription>
       </Alert>
     )}
     {cancelled && !failed && (

From 790c9ab77d1628e7c8be1b13002428a66bb16a1f Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 15 Sep 2026 16:25:01 -0700
Subject: [PATCH 59/67] test(ui): name the session column fixtures so the
 inline-object lint budget holds

---
 .../RequestLogsTableColumns.test.tsx          | 59 ++++++++-----------
 1 file changed, 25 insertions(+), 34 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
index d60b9ce30f1..626d290e0e8 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
@@ -75,14 +75,13 @@ describe("Cost column", () => {
   });
 
   it("does not label the per-call spend a session total when the aggregate is unavailable", () => {
-    renderRows([
-      logEntry({
-        request_id: "req-session-no-aggregate",
-        spend: 0.01,
-        session_id: "sess-1",
-        session_total_count: 3,
-      }),
-    ]);
+    const rowWithoutAggregate: Partial<LogEntry> = {
+      request_id: "req-session-no-aggregate",
+      spend: 0.01,
+      session_id: "sess-1",
+      session_total_count: 3,
+    };
+    renderRows([logEntry(rowWithoutAggregate)]);
 
     expect(screen.getByText("$0.010000")).toBeInTheDocument();
     expect(screen.queryByText("session total")).not.toBeInTheDocument();
@@ -90,16 +89,16 @@ describe("Cost column", () => {
 });
 
 describe("Duration column", () => {
+  const sessionRow: Partial<LogEntry> = {
+    request_id: "req-session-duration",
+    request_duration_ms: 1200,
+    session_id: "sess-1",
+    session_total_count: 3,
+  };
+
   it("shows the summed session duration, not the representative call's duration, for a multi-round session", () => {
-    renderRows([
-      logEntry({
-        request_id: "req-session-duration",
-        request_duration_ms: 1200,
-        session_id: "sess-1",
-        session_total_count: 3,
-        session_total_duration_ms: 5400,
-      }),
-    ]);
+    const aggregatedRow: Partial<LogEntry> = { ...sessionRow, session_total_duration_ms: 5400 };
+    renderRows([logEntry(aggregatedRow)]);
 
     expect(screen.getByText("5.40")).toBeInTheDocument();
     expect(screen.queryByText("1.20")).not.toBeInTheDocument();
@@ -107,29 +106,21 @@ describe("Duration column", () => {
   });
 
   it("does not label the per-call duration a session total when the aggregate is unavailable", () => {
-    renderRows([
-      logEntry({
-        request_id: "req-no-aggregate",
-        request_duration_ms: 1200,
-        session_id: "sess-3",
-        session_total_count: 3,
-      }),
-    ]);
+    renderRows([logEntry(sessionRow)]);
 
     expect(screen.getByText("1.20")).toBeInTheDocument();
     expect(screen.queryByText("session total")).not.toBeInTheDocument();
   });
 
   it("shows the call's own duration for a single-call session", () => {
-    renderRows([
-      logEntry({
-        request_id: "req-single-duration",
-        request_duration_ms: 1200,
-        session_id: "sess-2",
-        session_total_count: 1,
-        session_total_duration_ms: 1200,
-      }),
-    ]);
+    const singleCallRow: Partial<LogEntry> = {
+      ...sessionRow,
+      request_id: "req-single-duration",
+      session_id: "sess-2",
+      session_total_count: 1,
+      session_total_duration_ms: 1200,
+    };
+    renderRows([logEntry(singleCallRow)]);
 
     expect(screen.getByText("1.20")).toBeInTheDocument();
   });

From d61908e241483a6b968cc577020baeee65262b9b Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 23:28:01 +0000
Subject: [PATCH 60/67] refactor(proxy): type request_data on the alias rewrite
 helpers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/user_api_key_auth.py          | 2 +-
 litellm/proxy/common_utils/http_parsing_utils.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 15dd58baf35..4a35310272c 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -245,7 +245,7 @@ async def _normalize_claude_model(
 
 
 async def _resolve_router_settings_model_group_alias(
-    request_data: dict,  # mutable-ok: the request body is rewritten in place for every downstream reader
+    request_data: dict[str, object],  # mutable-ok: the request body is rewritten in place for every downstream reader
     valid_token: UserAPIKeyAuth,
     request: Request | None,
     route: str,
diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py
index a1dcc6e8ece..f5b6a0a766d 100644
--- a/litellm/proxy/common_utils/http_parsing_utils.py
+++ b/litellm/proxy/common_utils/http_parsing_utils.py
@@ -267,7 +267,7 @@ def _safe_set_request_parsed_body(
 
 
 def rewrite_request_model(
-    request_data: dict,  # mutable-ok: the request body is rewritten in place for every downstream reader
+    request_data: dict[str, object],  # mutable-ok: the request body is rewritten in place for every downstream reader
     request: Request | None,
     model: str,
 ) -> None:

From 771b2509d1ed2096da56325b0a6ee2a63f8b0a52 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 23:34:22 +0000
Subject: [PATCH 61/67] fix(proxy): reject mixed NIM model groups and strip the
 deployment model before the group in /nvidia_nim URLs

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../nvidia_nim/passthrough/transformation.py  | 24 ++++--
 litellm/proxy/auth/auth_utils.py              | 16 +---
 .../llm_passthrough_endpoints.py              | 13 +--
 ...t_nvidia_nim_passthrough_transformation.py | 80 +++++++++++++++++++
 .../proxy/auth/test_auth_utils.py             | 13 +++
 .../test_llm_pass_through_endpoints.py        | 26 ++++++
 6 files changed, 144 insertions(+), 28 deletions(-)

diff --git a/litellm/llms/nvidia_nim/passthrough/transformation.py b/litellm/llms/nvidia_nim/passthrough/transformation.py
index 6df9681cc81..7de1ce4d631 100644
--- a/litellm/llms/nvidia_nim/passthrough/transformation.py
+++ b/litellm/llms/nvidia_nim/passthrough/transformation.py
@@ -27,14 +27,26 @@ if TYPE_CHECKING:
 
 API_VERSION_SEGMENT: Final = re.compile(r"^v\d+$")
 NVIDIA_NIM_MODEL_PREFIX: Final = f"{LlmProviders.NVIDIA_NIM.value}/"
+NVIDIA_NIM_ROUTE_PREFIX: Final = re.compile(rf"^/{LlmProviders.NVIDIA_NIM.value}/", re.IGNORECASE)
+
+
+def is_nvidia_nim_deployment(deployment: DeploymentTypedDict) -> bool:
+    litellm_params: Final = deployment["litellm_params"]
+    return litellm_params.get("custom_llm_provider") == LlmProviders.NVIDIA_NIM.value or litellm_params.get(
+        "model", ""
+    ).startswith(NVIDIA_NIM_MODEL_PREFIX)
 
 
 def nvidia_nim_model_groups(deployments: Iterable[DeploymentTypedDict] | None) -> frozenset[str]:
-    return frozenset(
-        deployment["model_name"]
-        for deployment in deployments or ()
-        if deployment["litellm_params"].get("custom_llm_provider") == LlmProviders.NVIDIA_NIM.value
-        or deployment["litellm_params"].get("model", "").startswith(NVIDIA_NIM_MODEL_PREFIX)
+    listed: Final = tuple(deployments or ())
+    nim_groups: Final = frozenset(d["model_name"] for d in listed if is_nvidia_nim_deployment(d))
+    other_groups: Final = frozenset(d["model_name"] for d in listed if not is_nvidia_nim_deployment(d))
+    return nim_groups - other_groups
+
+
+def nvidia_nim_model_group_in_path(path: str, deployments: Iterable[DeploymentTypedDict] | None) -> str | None:
+    return nvidia_nim_router_model_in_endpoint(
+        NVIDIA_NIM_ROUTE_PREFIX.sub("", path), nvidia_nim_model_groups(deployments)
     )
 
 
@@ -79,7 +91,7 @@ class NvidiaNimPassthroughConfig(BasePassthroughConfig):
         base_target_url: Final = self.get_api_base(api_base)
         if base_target_url is None:
             raise ValueError("NVIDIA NIM api base not found: set `api_base` on the deployment or NVIDIA_NIM_API_BASE")
-        native_endpoint: Final = strip_leading_model_segment(endpoint, (model_group_from(litellm_params), model))
+        native_endpoint: Final = strip_leading_model_segment(endpoint, (model, model_group_from(litellm_params)))
         root: Final = without_repeated_version_prefix(base_target_url, native_endpoint)
         return (self.format_url(native_endpoint, root, request_query_params), root)
 
diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index dfce3ab206b..e69b0c98ee0 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -28,10 +28,7 @@ from litellm.litellm_core_utils.url_utils import (
     validate_url,
 )
 from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
-from litellm.llms.nvidia_nim.passthrough.transformation import (
-    nvidia_nim_model_groups,
-    nvidia_nim_router_model_in_endpoint,
-)
+from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
 from litellm.proxy._types import *
 from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
@@ -2045,19 +2042,14 @@ def get_model_from_request(
         return model if azure_model is None else azure_model
 
     if route.lower().startswith("/nvidia_nim/"):
-        nvidia_nim_model: Final = _router_model_from_nvidia_nim_route(route, llm_router)
+        nvidia_nim_model: Final = (
+            nvidia_nim_model_group_in_path(route, llm_router.get_model_list()) if llm_router else None
+        )
         return model if nvidia_nim_model is None else nvidia_nim_model
 
     return model
 
 
-def _router_model_from_nvidia_nim_route(route: str, llm_router: Router | None) -> str | None:
-    if llm_router is None:
-        return None
-    endpoint: Final = re.sub(r"^/nvidia_nim/", "", route, flags=re.IGNORECASE)
-    return nvidia_nim_router_model_in_endpoint(endpoint, nvidia_nim_model_groups(llm_router.get_model_list()))
-
-
 def _router_model_from_azure_route(route: str, llm_router: Router | None) -> str | None:
     if llm_router is None:
         return None
diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index 65a18c1603e..4fc610f068f 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -36,10 +36,7 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.azure.passthrough.transformation import foreign_azure_deployment
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
-from litellm.llms.nvidia_nim.passthrough.transformation import (
-    nvidia_nim_model_groups,
-    nvidia_nim_router_model_in_endpoint,
-)
+from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.passthrough.main import AsyncPassthroughStreamingResponse
 from litellm.proxy._types import *
@@ -1654,15 +1651,11 @@ async def relay_nvidia_nim_request(
     request_body: Mapping[str, object],
     user_api_key_dict: UserAPIKeyAuth,
 ) -> Response:
-    model_group: Final = (
-        nvidia_nim_router_model_in_endpoint(endpoint, nvidia_nim_model_groups(llm_router.get_model_list()))
-        if llm_router
-        else None
-    )
+    model_group: Final = nvidia_nim_model_group_in_path(endpoint, llm_router.get_model_list()) if llm_router else None
     if llm_router is None or model_group is None:
         rejection: Final[RelayRejection] = {
             "error": "no NVIDIA NIM model group in the path; call /nvidia_nim/{model_group}/v1/infer with a model "
-            "from your `model_list` whose `model` starts with `nvidia_nim/`"
+            "group from your `model_list` whose deployments all use `nvidia_nim/` models"
         }
         raise HTTPException(status_code=400, detail=rejection)
 
diff --git a/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py b/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py
index 87de4a795e0..906d6c2b614 100644
--- a/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py
+++ b/tests/test_litellm/llms/nvidia_nim/passthrough/test_nvidia_nim_passthrough_transformation.py
@@ -8,6 +8,8 @@ import litellm
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.llms.nvidia_nim.passthrough.transformation import (
     NvidiaNimPassthroughConfig,
+    nvidia_nim_model_group_in_path,
+    nvidia_nim_model_groups,
     nvidia_nim_router_model_in_endpoint,
 )
 from litellm.types.utils import LlmProviders
@@ -53,6 +55,12 @@ def test_provider_config_manager_resolves_nvidia_nim_passthrough_config():
         (f"{NIM_BASE}/v2", "v1/infer", {}, f"{NIM_BASE}/v2/v1/infer"),
         (f"{NIM_BASE}/infer", "infer", {}, f"{NIM_BASE}/infer/infer"),
         (NIM_BASE, "nvidia/nemoretriever-page-elements-v2/v1/infer", {}, f"{NIM_BASE}/v1/infer"),
+        (
+            NIM_BASE,
+            "nvidia/nemoretriever-page-elements-v2/v1/infer",
+            {"litellm_metadata": {"model_group": "nvidia"}},
+            f"{NIM_BASE}/v1/infer",
+        ),
     ],
 )
 def test_relay_url_strips_the_model_group_and_never_doubles_the_api_version(
@@ -161,6 +169,45 @@ def test_router_model_in_endpoint_takes_the_longest_leading_model_group(endpoint
     assert nvidia_nim_router_model_in_endpoint(endpoint, frozenset(router_models)) == expected
 
 
+def _deployment(model_name: str, model: str, custom_llm_provider: str | None = None):
+    litellm_params = (
+        {"model": model}
+        if custom_llm_provider is None
+        else {"model": model, "custom_llm_provider": custom_llm_provider}
+    )
+    return {"model_name": model_name, "litellm_params": litellm_params}
+
+
+MIXED_DEPLOYMENTS = (
+    _deployment("nim-page", "nvidia_nim/nvidia/nemoretriever-page-elements-v2"),
+    _deployment("nim-table", "nvidia/nemoretriever-table-structure-v1", custom_llm_provider="nvidia_nim"),
+    _deployment("mixed", "nvidia_nim/nvidia/nemoretriever-page-elements-v2"),
+    _deployment("mixed", "openai/gpt-4o"),
+    _deployment("gpt-4o", "openai/gpt-4o"),
+)
+
+
+def test_model_groups_only_admit_groups_whose_every_deployment_is_nim_backed():
+    assert nvidia_nim_model_groups(MIXED_DEPLOYMENTS) == frozenset({"nim-page", "nim-table"})
+    assert nvidia_nim_model_groups(None) == frozenset()
+
+
+@pytest.mark.parametrize(
+    "path, expected",
+    [
+        ("/nvidia_nim/nim-page/v1/infer", "nim-page"),
+        ("/NVIDIA_NIM/nim-table/v1/infer", "nim-table"),
+        ("nim-page/v1/infer", "nim-page"),
+        ("/nvidia_nim/mixed/v1/infer", None),
+        ("mixed/v1/infer", None),
+        ("/nvidia_nim/gpt-4o/v1/infer", None),
+        ("/nvidia_nim/v1/infer", None),
+    ],
+)
+def test_model_group_in_path_resolves_the_same_nim_only_groups_for_routes_and_endpoints(path, expected):
+    assert nvidia_nim_model_group_in_path(path, MIXED_DEPLOYMENTS) == expected
+
+
 @pytest.mark.parametrize("request_data, expected", [({"stream": True}, True), ({"stream": False}, False), ({}, False)])
 def test_is_streaming_request_reads_the_stream_flag(request_data, expected):
     assert NvidiaNimPassthroughConfig().is_streaming_request("v1/infer", request_data) is expected
@@ -214,3 +261,36 @@ async def test_object_detection_relay_sends_the_native_body_unchanged_to_v1_infe
     assert response.status_code == 200
     assert response.headers["x-nim"] == "1"
     assert response.json() == {"data": [{"index": 0}, {"index": 1}]}
+
+
+@pytest.mark.asyncio
+async def test_router_relay_reaches_v1_infer_when_the_group_name_is_a_leading_segment_of_the_model_id():
+    upstream_requests: list[httpx.Request] = []
+
+    def nim(request: httpx.Request) -> httpx.Response:
+        upstream_requests.append(request)
+        return httpx.Response(200, json={"data": [{"index": 0}]})
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(nim))
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "nvidia",
+                "litellm_params": {
+                    "model": "nvidia_nim/nvidia/nemoretriever-page-elements-v2",
+                    "api_base": NIM_BASE,
+                    "api_key": "nvapi-secret",
+                },
+            }
+        ]
+    )
+
+    response = await router.allm_passthrough_route(
+        model="nvidia", endpoint="nvidia/v1/infer", method="POST", json=dict(INFER_BODY), client=client
+    )
+
+    (sent,) = upstream_requests
+    assert str(sent.url) == f"{NIM_BASE}/v1/infer"
+    assert json.loads(sent.content) == INFER_BODY
+    assert response.status_code == 200
diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py
index 0841f82227a..8e592b61f1c 100644
--- a/tests/test_litellm/proxy/auth/test_auth_utils.py
+++ b/tests/test_litellm/proxy/auth/test_auth_utils.py
@@ -848,6 +848,18 @@ def _nvidia_nim_relay_router():
                 "model_name": "gpt-4o",
                 "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"},
             },
+            {
+                "model_name": "detect",
+                "litellm_params": {
+                    "model": "nvidia_nim/nvidia/nemoretriever-page-elements-v2",
+                    "api_base": "http://nim-a.internal:8000",
+                    "api_key": "k",
+                },
+            },
+            {
+                "model_name": "detect",
+                "litellm_params": {"model": "openai/gpt-4o", "api_key": "k"},
+            },
         ]
     )
 
@@ -873,6 +885,7 @@ NIM_INFER_BODY = {"input": [{"type": "image_url", "url": "data:image/png;base64,
         ("/nvidia_nim/unknown-group/v1/infer", NIM_INFER_BODY, None),
         ("/nvidia_nim/nim-page-elements-v2/v1/infer", NIM_INFER_BODY, None),
         ("/nvidia_nim/gpt-4o/v1/infer", NIM_INFER_BODY, None),
+        ("/nvidia_nim/detect/v1/infer", NIM_INFER_BODY, None),
     ],
 )
 def test_get_model_from_request_nvidia_nim_relay_routes_use_the_model_group_in_the_path(route, request_data, expected):
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
index 4e73a33ecd9..62ecbbbcf51 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
@@ -5498,6 +5498,32 @@ class TestNvidiaNimProxyRoute:
         assert exc_info.value.status_code == 400
         assert captured == []
 
+    @pytest.mark.asyncio
+    async def test_a_group_mixing_nim_and_other_deployments_is_rejected_before_any_upstream_call(self):
+        from fastapi import HTTPException
+
+        captured: list[dict] = []
+
+        class MixedRouter:
+            def get_model_list(self):
+                return [
+                    {
+                        "model_name": "detect",
+                        "litellm_params": {"model": "nvidia_nim/nvidia/nemoretriever-page-elements-v2"},
+                    },
+                    {"model_name": "detect", "litellm_params": {"model": "openai/gpt-4o"}},
+                ]
+
+            async def allm_passthrough_route(self, **kwargs):
+                captured.append(kwargs)
+                return httpx.Response(200, json={"data": []})
+
+        with pytest.raises(HTTPException) as exc_info:
+            await self._relay(MixedRouter(), "detect/v1/infer", NIM_INFER_BODY)
+
+        assert exc_info.value.status_code == 400
+        assert captured == []
+
     @pytest.mark.asyncio
     async def test_no_router_is_rejected_before_any_upstream_call(self):
         from fastapi import HTTPException

From 74a2c2eab1f86ada6bc4ccc55dfab64bd6d39219 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 23:34:31 +0000
Subject: [PATCH 62/67] refactor(vertex_ai): tighten Interactions route match
 and type the usage boundary

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/cost_calculator.py                    |  4 +---
 .../llms/vertex_ai/videos/transformation.py   |  1 -
 .../vertex_passthrough_logging_handler.py     | 20 ++++++++++++++-----
 .../pass_through_endpoints/success_handler.py |  4 +---
 .../test_pass_through_endpoints.py            |  7 +++++++
 5 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index f81eb1539fd..e6bb8da4c6a 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -1460,7 +1460,6 @@ def completion_cost(
                     duration_seconds: float | None = None
                     video_resolution: str | None = None
                     provider_reported_cost: float | None = None
-                    video_count: int = 1
                     if completion_response is not None and usage_obj:
                         # Handle both dict and Pydantic Usage object
                         if isinstance(usage_obj, dict):
@@ -1475,8 +1474,7 @@ def completion_cost(
                             _vc = getattr(usage_obj, "video_count", None)
                         if _vr is not None:
                             video_resolution = str(_vr).strip().lower()
-                        if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1:
-                            video_count = _vc
+                        video_count = _vc if isinstance(_vc, int) and not isinstance(_vc, bool) and _vc > 1 else 1
 
                         if _video_model_info is None and provider_reported_cost is not None:
                             return float(provider_reported_cost)
diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py
index 6525c6fa235..dc9caa13224 100644
--- a/litellm/llms/vertex_ai/videos/transformation.py
+++ b/litellm/llms/vertex_ai/videos/transformation.py
@@ -71,7 +71,6 @@ def _parse_veo_operation(raw_response: httpx.Response) -> _VeoOperation:
 
 
 def veo_video_count_from_parameters(parameters: Mapping[str, object]) -> int | None:
-    """Number of videos Veo generates for one request (``parameters.sampleCount``)."""
     sample_count: Final = parameters.get("sampleCount")
     if isinstance(sample_count, bool) or not isinstance(sample_count, int) or sample_count < 1:
         return None
diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
index 0c4bc2a5181..cd226e80c6e 100644
--- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
+++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast
 from urllib.parse import urlparse
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -53,6 +54,9 @@ else:
 
 EndpointType = Any
 
+_VERTEX_INTERACTIONS_PATH: Final = re.compile(r"/projects/[^/]+/locations/[^/]+/interactions/?$")
+_INTERACTIONS_RESPONSE_BODY: Final = TypeAdapter(dict[str, object])
+
 
 def _interactions_model(
     response_body: Mapping[str, object],
@@ -72,18 +76,22 @@ class VertexPassthroughLoggingHandler:
     def is_interactions_route(url_route: str) -> bool:
         return urlparse(url_route).path.rstrip("/").endswith("/interactions")
 
+    @staticmethod
+    def is_vertex_interactions_route(url_route: str) -> bool:
+        return _VERTEX_INTERACTIONS_PATH.search(urlparse(url_route).path) is not None
+
     @staticmethod
     def interactions_passthrough_handler(
         httpx_response: httpx.Response,
         request_body: Mapping[str, object] | None,
         logging_obj: LiteLLMLoggingObj,
-        kwargs: dict,
+        kwargs: dict[str, object],
         start_time: datetime,
         end_time: datetime,
         custom_llm_provider: Literal["vertex_ai", "gemini"],
         vertex_location: str | None,
     ) -> PassThroughEndpointLoggingTypedDict:
-        response_body: Final[Mapping[str, object]] = httpx_response.json()
+        response_body: Final = _INTERACTIONS_RESPONSE_BODY.validate_python(httpx_response.json())
         usage_object: Final = response_body.get("usage")
         model: Final = _interactions_model(response_body, request_body)
         if model is None or not InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_object):
@@ -95,6 +103,7 @@ class VertexPassthroughLoggingHandler:
                 cast(Mapping[str, Any], usage_object)
             ),
         )
+        logging_obj.custom_llm_provider = custom_llm_provider
         logging_kwargs: Final = (
             VertexPassthroughLoggingHandler._create_vertex_response_logging_payload_for_generate_content(
                 litellm_model_response=litellm_model_response,
@@ -107,9 +116,10 @@ class VertexPassthroughLoggingHandler:
                 vertex_location=vertex_location,
             )
         )
-        logging_kwargs["custom_llm_provider"] = custom_llm_provider
-        logging_obj.custom_llm_provider = custom_llm_provider
-        return {"result": litellm_model_response, "kwargs": logging_kwargs}
+        return {
+            "result": litellm_model_response,
+            "kwargs": {**logging_kwargs, "custom_llm_provider": custom_llm_provider},
+        }
 
     @staticmethod
     def vertex_passthrough_handler(
diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py
index dd8bd17eee9..76a471302f4 100644
--- a/litellm/proxy/pass_through_endpoints/success_handler.py
+++ b/litellm/proxy/pass_through_endpoints/success_handler.py
@@ -363,9 +363,7 @@ class PassThroughEndpointLogging:
             return True
         if any(resource in url_route for resource in self.TRACKED_VERTEX_RESOURCE_ROUTES):
             return True
-        return "/locations/" in urlparse(url_route).path and VertexPassthroughLoggingHandler.is_interactions_route(
-            url_route
-        )
+        return VertexPassthroughLoggingHandler.is_vertex_interactions_route(url_route)
 
     def is_anthropic_route(self, url_route: str):
         for route in self.TRACKED_ANTHROPIC_ROUTES:
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
index 988c4782f6c..cc1d408176c 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
@@ -510,6 +510,13 @@ def test_interactions_create_routes_are_tracked_for_vertex_and_gemini():
     assert handler.is_vertex_route(vertex_create) is True
     assert handler.is_vertex_route(f"{vertex_create}/abc123") is False
     assert handler.is_vertex_route("https://upstream.example.com/api/interactions") is False
+    assert handler.is_vertex_route("https://upstream.example.com/locations/eu/interactions") is False
+    assert (
+        handler.is_vertex_route(
+            "https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/interactions"
+        )
+        is True
+    )
 
     assert handler.is_gemini_route(gemini_create, custom_llm_provider="gemini") is True
     assert handler.is_gemini_route(f"{gemini_create}/abc123", custom_llm_provider="gemini") is False

From 5b54bf2328c53244d9509256defbc9f06f54d97a Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Tue, 15 Sep 2026 23:36:22 +0000
Subject: [PATCH 63/67] refactor(model_info): restore provider-neutral Gemini
 chat baseline

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ...odel_prices_and_context_window_backup.json |  4 +--
 model_prices_and_context_window.json          |  4 +--
 .../test_fallback_generalizations.py          | 26 +++++++++++--------
 3 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 9569f29aa0e..cc4c7feffad 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -58152,8 +58152,8 @@
             },
             {
                 "name": "gemini-chat-baseline",
-                "pattern": "^(?:gemini|vertex_ai)/gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2\\.[5-9]|[3-9](?:\\.\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "A first-party Gemini text-chat id at 2.5 or higher, as gemini/<id> or vertex_ai/<id> (get_model_info tries the provider-prefixed name first, so a bare id under those providers matches too): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. Scoped to Google's own namespaces because resellers wrap the same models differently (one exposes them as mode responses without reasoning), so nothing below is certain there. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. 2.5 is the floor: every first-party text-chat Gemini from 2.5 onward in this map carries all of these fields, 1.5 and 2.0 do not and are fully mapped. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
                 "model_info": {
                     "mode": "chat",
                     "supports_reasoning": true,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 9569f29aa0e..cc4c7feffad 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -58152,8 +58152,8 @@
             },
             {
                 "name": "gemini-chat-baseline",
-                "pattern": "^(?:gemini|vertex_ai)/gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2\\.[5-9]|[3-9](?:\\.\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "A first-party Gemini text-chat id at 2.5 or higher, as gemini/<id> or vertex_ai/<id> (get_model_info tries the provider-prefixed name first, so a bare id under those providers matches too): gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. Scoped to Google's own namespaces because resellers wrap the same models differently (one exposes them as mode responses without reasoning), so nothing below is certain there. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. 2.5 is the floor: every first-party text-chat Gemini from 2.5 onward in this map carries all of these fields, 1.5 and 2.0 do not and are fully mapped. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
                 "model_info": {
                     "mode": "chat",
                     "supports_reasoning": true,
diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
index a4fbd79187e..71e6e20b1a4 100644
--- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
+++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
@@ -457,12 +457,19 @@ def shipped_cost_map(monkeypatch):
         ("gemini-3.9-flash-lite-preview-09-2026", "vertex_ai"),
         ("vertex_ai/gemini-4-pro", None),
         ("gemini-4-pro-preview-customtools", "gemini"),
+        ("google/gemini-4-pro", "openrouter"),
+        ("google/gemini-4-pro", "deepinfra"),
+        ("google/gemini-4-pro", "vercel_ai_gateway"),
+        ("google.gemini-4-pro", "oci"),
+        ("databricks-gemini-4-1-pro", "databricks"),
     ],
 )
-def test_shipped_gemini_chat_baseline_resolves_first_party_unmapped_ids(shipped_cost_map, model, provider):
+def test_shipped_gemini_chat_baseline_resolves_unmapped_ids(shipped_cost_map, model, provider):
     assert model not in litellm.model_cost
     if provider == "gemini":
         assert f"gemini/{model}" not in litellm.model_cost
+    elif provider in {"openrouter", "deepinfra", "vercel_ai_gateway", "oci", "databricks"}:
+        assert f"{provider}/{model}" not in litellm.model_cost
 
     info = litellm.get_model_info(model, custom_llm_provider=provider)
     assert info["litellm_provider"] == (provider or model.split("/")[0])
@@ -481,16 +488,11 @@ def test_shipped_gemini_chat_baseline_resolves_first_party_unmapped_ids(shipped_
     assert not info.get("output_cost_per_token")
 
 
-def test_shipped_gemini_chat_baseline_skips_reseller_namespaces(shipped_cost_map):
-    for model, provider in (
-        ("google/gemini-4-pro", "perplexity"),
-        ("google/gemini-4-pro", "openrouter"),
-        ("databricks-gemini-4-1-pro", "databricks"),
-    ):
-        with pytest.raises(Exception, match="isn't mapped yet"):
-            litellm.get_model_info(model, custom_llm_provider=provider)
-
-    assert match_capability_generalizations("google/gemini-4-pro") is None
+def test_shipped_gemini_chat_baseline_loses_to_perplexity_exact_entries(shipped_cost_map):
+    info = litellm.get_model_info("google/gemini-2.5-pro", custom_llm_provider="perplexity")
+    entry = litellm.model_cost["perplexity/google/gemini-2.5-pro"]
+    assert info["mode"] == "responses"
+    assert entry["supports_reasoning"] is False
 
 
 def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cost_map):
@@ -506,6 +508,8 @@ def test_shipped_gemini_chat_baseline_skips_non_chat_and_pre_2_5_ids(shipped_cos
         "gemini/gemini-4-flashy",
         "gemini/gemini-4-flash-transcribe",
         "gemini/gemini-4-flash-live-translate-preview",
+        "databricks-gemini-3-1-flash-image",
+        "openrouter/google/gemini-2.0-flash-001",
     ):
         assert match_capability_generalizations(model) is None, model
 

From 945adfb603bad0ee0f9a5054196e0b8beeb05a39 Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Tue, 15 Sep 2026 23:49:18 +0000
Subject: [PATCH 64/67] refactor(model_info): support multi-digit Gemini majors

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/model_prices_and_context_window_backup.json           | 4 ++--
 model_prices_and_context_window.json                          | 4 ++--
 .../litellm_core_utils/test_fallback_generalizations.py       | 2 ++
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index cc4c7feffad..5537f13f5fd 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -58152,8 +58152,8 @@
             },
             {
                 "name": "gemini-chat-baseline",
-                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|(?:[3-9]|[1-9]\\d)(?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number. Source check 2026-09-15: all 45 first-party 2.5+ text-chat entries in this map carry every field below, and the OpenRouter (openrouter.ai/api/v1/models), Vercel AI Gateway (ai-gateway.vercel.sh/v1/models), DeepInfra (api.deepinfra.com/models/list), OCI and Databricks model docs list reasoning, tools and image input for the same models.",
                 "model_info": {
                     "mode": "chat",
                     "supports_reasoning": true,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index cc4c7feffad..5537f13f5fd 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -58152,8 +58152,8 @@
             },
             {
                 "name": "gemini-chat-baseline",
-                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
-                "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number.",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|(?:[3-9]|[1-9]\\d)(?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number. Source check 2026-09-15: all 45 first-party 2.5+ text-chat entries in this map carry every field below, and the OpenRouter (openrouter.ai/api/v1/models), Vercel AI Gateway (ai-gateway.vercel.sh/v1/models), DeepInfra (api.deepinfra.com/models/list), OCI and Databricks model docs list reasoning, tools and image input for the same models.",
                 "model_info": {
                     "mode": "chat",
                     "supports_reasoning": true,
diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
index 71e6e20b1a4..4e7d7e85a8c 100644
--- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
+++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
@@ -458,10 +458,12 @@ def shipped_cost_map(monkeypatch):
         ("vertex_ai/gemini-4-pro", None),
         ("gemini-4-pro-preview-customtools", "gemini"),
         ("google/gemini-4-pro", "openrouter"),
+        ("google/gemini-12.5-flash-lite", "openrouter"),
         ("google/gemini-4-pro", "deepinfra"),
         ("google/gemini-4-pro", "vercel_ai_gateway"),
         ("google.gemini-4-pro", "oci"),
         ("databricks-gemini-4-1-pro", "databricks"),
+        ("gemini-10-pro", "gemini"),
     ],
 )
 def test_shipped_gemini_chat_baseline_resolves_unmapped_ids(shipped_cost_map, model, provider):

From c9dd4b44f8d696d3d703f31b77f6257990db3ecf Mon Sep 17 00:00:00 2001
From: kerry <kerry@berri.ai>
Date: Tue, 15 Sep 2026 23:51:56 +0000
Subject: [PATCH 65/67] revert(model_info): keep Gemini baseline majors
 single-digit

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/model_prices_and_context_window_backup.json             | 2 +-
 model_prices_and_context_window.json                            | 2 +-
 .../litellm_core_utils/test_fallback_generalizations.py         | 2 --
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 5537f13f5fd..36b4a2c308a 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -58152,7 +58152,7 @@
             },
             {
                 "name": "gemini-chat-baseline",
-                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|(?:[3-9]|[1-9]\\d)(?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
                 "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number. Source check 2026-09-15: all 45 first-party 2.5+ text-chat entries in this map carry every field below, and the OpenRouter (openrouter.ai/api/v1/models), Vercel AI Gateway (ai-gateway.vercel.sh/v1/models), DeepInfra (api.deepinfra.com/models/list), OCI and Databricks model docs list reasoning, tools and image input for the same models.",
                 "model_info": {
                     "mode": "chat",
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 5537f13f5fd..36b4a2c308a 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -58152,7 +58152,7 @@
             },
             {
                 "name": "gemini-chat-baseline",
-                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|(?:[3-9]|[1-9]\\d)(?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
+                "pattern": "gemini-(?!.*(?:-tts|-image|-live|-audio|-embedding|-computer-use|-robotics|-transcribe|-translate))(?:2[.-][5-9]|[3-9](?:[.-]\\d{1,2})?)-(?:pro|flash)(?:-lite)?(?![a-z])",
                 "description": "Any Gemini text-chat id at 2.5 or higher under any namespace, including bare ids, gemini/, vertex_ai/, openrouter/google/, deepinfra/google/, vercel_ai_gateway/google/, oci/google., and databricks-gemini-<major>-<minor>: gemini-<major>[.minor]-(pro|flash)[-lite] with any trailing preview, date or variant tag. The capability flags were verified against each of those providers' own catalogs and docs. The lookahead excludes the tts, image, live, audio, embedding, computer-use, robotics, transcribe and translate lines, which are different modes with different capabilities. Provider-specific deviations, such as Perplexity's Agent API serving these as mode responses, are carried by their exact map entries, which always win over this rule. Carries no token limits or pricing, so those stay on the standard unmapped behavior rather than a guessed number. Source check 2026-09-15: all 45 first-party 2.5+ text-chat entries in this map carry every field below, and the OpenRouter (openrouter.ai/api/v1/models), Vercel AI Gateway (ai-gateway.vercel.sh/v1/models), DeepInfra (api.deepinfra.com/models/list), OCI and Databricks model docs list reasoning, tools and image input for the same models.",
                 "model_info": {
                     "mode": "chat",
diff --git a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
index 4e7d7e85a8c..71e6e20b1a4 100644
--- a/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
+++ b/tests/test_litellm/litellm_core_utils/test_fallback_generalizations.py
@@ -458,12 +458,10 @@ def shipped_cost_map(monkeypatch):
         ("vertex_ai/gemini-4-pro", None),
         ("gemini-4-pro-preview-customtools", "gemini"),
         ("google/gemini-4-pro", "openrouter"),
-        ("google/gemini-12.5-flash-lite", "openrouter"),
         ("google/gemini-4-pro", "deepinfra"),
         ("google/gemini-4-pro", "vercel_ai_gateway"),
         ("google.gemini-4-pro", "oci"),
         ("databricks-gemini-4-1-pro", "databricks"),
-        ("gemini-10-pro", "gemini"),
     ],
 )
 def test_shipped_gemini_chat_baseline_resolves_unmapped_ids(shipped_cost_map, model, provider):

From 65f9d9bbbd4a842698793b6ecdc98155aecb1427 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Tue, 15 Sep 2026 23:57:16 +0000
Subject: [PATCH 66/67] test(proxy): use the module-level HTTPException import
 in the /nvidia_nim route tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../test_llm_pass_through_endpoints.py                      | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
index 62ecbbbcf51..73e6ceabdb6 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py
@@ -5484,8 +5484,6 @@ class TestNvidiaNimProxyRoute:
         ["v1/infer", "unknown-group/v1/infer", "nim-page-elements-v2/v1/infer", "gpt-4o/v1/infer"],
     )
     async def test_path_without_a_nim_model_group_is_rejected_before_any_upstream_call(self, endpoint):
-        from fastapi import HTTPException
-
         captured: list[dict] = []
         router = self._recording_router(
             captured,
@@ -5500,8 +5498,6 @@ class TestNvidiaNimProxyRoute:
 
     @pytest.mark.asyncio
     async def test_a_group_mixing_nim_and_other_deployments_is_rejected_before_any_upstream_call(self):
-        from fastapi import HTTPException
-
         captured: list[dict] = []
 
         class MixedRouter:
@@ -5526,8 +5522,6 @@ class TestNvidiaNimProxyRoute:
 
     @pytest.mark.asyncio
     async def test_no_router_is_rejected_before_any_upstream_call(self):
-        from fastapi import HTTPException
-
         with pytest.raises(HTTPException) as exc_info:
             await self._relay(None, "nim-page-elements/v1/infer", NIM_INFER_BODY)
 

From f25272bfa0d1bd0cdcbf68a59ac047359498a7ad Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Wed, 16 Sep 2026 00:05:17 +0000
Subject: [PATCH 67/67] fix(gateway): expose /nvidia_nim/ on the gateway
 data-plane allowlist

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 gateway/routes/allowlist.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py
index 3733072a948..099c6d5179f 100644
--- a/gateway/routes/allowlist.py
+++ b/gateway/routes/allowlist.py
@@ -96,6 +96,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/langfuse/",
     "/vllm/",
     "/mistral/",
+    "/nvidia_nim/",
     "/groq/",
     "/voyage/",
     "/cursor/",