From 55c7872496b5568bc259fc88c0645f702af1576e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 12:57:06 -0700
Subject: [PATCH 01/27] fix(proxy-extras): rebuild indexes left INVALID by a
 migration deadlock

Two replicas racing prisma migrate deploy can deadlock, and the loser dies
mid CREATE INDEX CONCURRENTLY, leaving the index INVALID. The retried
migration's IF NOT EXISTS then skips it, so the planner never uses it.

After migrations succeed, look for INVALID indexes on LiteLLM tables and
have one replica (advisory try-lock) REINDEX INDEX CONCURRENTLY each of
them, dropping _ccnew/_ccold leftovers of an interrupted rebuild instead.
The repair never blocks startup: a failed rebuild is logged and retried
on the next boot. Also encode DATABASE_URL query values with quote instead
of quote_plus so options=-c%20... reaches psycopg intact.
---
 .../litellm_proxy_extras/utils.py             | 128 +++++++++-
 .../test_invalid_index_repair.py              | 238 ++++++++++++++++++
 2 files changed, 359 insertions(+), 7 deletions(-)
 create mode 100644 tests/proxy_migration_tests/test_invalid_index_repair.py

diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py
index b8032dd0d28..df019d78078 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/utils.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py
@@ -6,18 +6,23 @@ import shutil
 import subprocess
 import tempfile
 import time
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Final, Optional
 
 from litellm_proxy_extras._logging import logger
-from litellm_proxy_extras.replica_identity import (
-    REPLICA_IDENTITY_FULL_ENV_VAR,
-    apply_replica_identity_full,
-)
 from litellm_proxy_extras.prisma_toolchain import (
     ensure_prisma_toolchain,
     prisma_command_timeout,
 )
+from litellm_proxy_extras.replica_identity import (
+    REPLICA_IDENTITY_FULL_ENV_VAR,
+    apply_replica_identity_full,
+)
+
+if TYPE_CHECKING:
+    import psycopg
+    import psycopg.sql
 
 
 def str_to_bool(value: Optional[str]) -> bool:
@@ -40,6 +45,29 @@ def _get_prisma_env() -> dict:
 
 _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
+INDEX_REPAIR_ADVISORY_LOCK_KEY: Final = int.from_bytes(b"litellm", "big")
+_TRANSIENT_INDEX_SUFFIX_RE: Final = re.compile(r"_cc(?:new|old)\d*$")
+_INVALID_LITELLM_INDEXES_SQL: Final = (
+    "SELECT n.nspname, c.relname, pg_size_pretty(pg_table_size(t.oid)) "
+    "FROM pg_index i "
+    "JOIN pg_class c ON c.oid = i.indexrelid "
+    "JOIN pg_class t ON t.oid = i.indrelid "
+    "JOIN pg_namespace n ON n.oid = t.relnamespace "
+    "WHERE NOT i.indisvalid "
+    "  AND c.relkind = 'i' "
+    "  AND n.nspname = %s "
+    "  AND t.relname LIKE %s "
+    "  AND NOT EXISTS (SELECT 1 FROM pg_constraint k WHERE k.conindid = i.indexrelid) "
+    "ORDER BY c.relname"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class _InvalidIndex:
+    schema: str
+    name: str
+    table_size: str
+
 _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
 _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
     r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
@@ -557,7 +585,7 @@ class ProxyExtrasDBManager:
     def _strip_prisma_query_params(url: str) -> str:
         """Remove Prisma-specific query params (connection_limit, pool_timeout,
         schema, etc.) from DATABASE_URL so psycopg can parse it."""
-        from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
+        from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
 
         parsed = urlparse(url)
         if not parsed.query:
@@ -578,7 +606,7 @@ class ProxyExtrasDBManager:
             "target_session_attrs",
         }
         kept = [(k, v) for k, v in parse_qsl(parsed.query) if k in libpq_params]
-        return urlunparse(parsed._replace(query=urlencode(kept)))
+        return urlunparse(parsed._replace(query=urlencode(kept, quote_via=quote)))
 
     @staticmethod
     def _warn_if_db_ahead_of_head(migrations_dir: str) -> None:
@@ -652,6 +680,91 @@ class ProxyExtrasDBManager:
             ", ".join(sorted_hostile[:5]) + (" ..." if len(sorted_hostile) > 5 else ""),
         )
 
+    @staticmethod
+    def _invalid_litellm_indexes(
+        conn: "psycopg.Connection[tuple[str, str, str]]", schema: str
+    ) -> tuple[_InvalidIndex, ...]:
+        rows: Final = conn.execute(_INVALID_LITELLM_INDEXES_SQL, (schema, "LiteLLM\\_%")).fetchall()
+        return tuple(_InvalidIndex(*row) for row in rows)
+
+    @staticmethod
+    def _index_repair(index: _InvalidIndex) -> tuple["psycopg.sql.Composed", str]:
+        from psycopg import sql
+
+        target: Final = sql.Identifier(index.schema, index.name)
+        if _TRANSIENT_INDEX_SUFFIX_RE.search(index.name):
+            return sql.SQL("DROP INDEX CONCURRENTLY IF EXISTS {}").format(target), "Dropped leftover"
+        return sql.SQL("REINDEX INDEX CONCURRENTLY {}").format(target), "Rebuilt"
+
+    @staticmethod
+    def _repair_index(conn: "psycopg.Connection[tuple[str, str, str]]", index: _InvalidIndex) -> None:
+        import psycopg
+
+        statement, action = ProxyExtrasDBManager._index_repair(index)
+        try:
+            conn.execute(statement)
+        except psycopg.Error as e:
+            logger.warning(
+                "Could not repair invalid index %s.%s, will retry on the next startup. "
+                "If this keeps happening, run `%s` by hand as the index owner. Error: %s",
+                index.schema,
+                index.name,
+                statement.as_string(conn),
+                e,
+            )
+            return
+        logger.info("%s invalid index %s.%s", action, index.schema, index.name)
+
+    @staticmethod
+    def repair_invalid_indexes(lock_timeout: str = "30s") -> bool:
+        """Rebuild LiteLLM indexes an interrupted CREATE INDEX CONCURRENTLY left
+        INVALID (a migration deadlock between replicas is the usual cause; the
+        retried migration skips them because of IF NOT EXISTS). Never raises:
+        returns True when no invalid index remains, False when the repair was
+        skipped or failed and will be retried on the next startup."""
+        database_url: Final = os.getenv("DATABASE_URL")
+        if not database_url:
+            return False
+
+        try:
+            import psycopg
+            from psycopg import sql
+        except ImportError:
+            logger.warning(
+                "psycopg is not installed; skipping the invalid index check. "
+                "Install the litellm[extra_proxy] extra, which includes psycopg."
+            )
+            return False
+
+        schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
+        cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        try:
+            with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
+                conn.execute("SET statement_timeout = 0")
+                conn.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(lock_timeout)))
+                found: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
+                if not found:
+                    return True
+                logger.warning(
+                    "Found %d invalid index(es) left by an interrupted CREATE INDEX "
+                    "CONCURRENTLY, rebuilding: %s",
+                    len(found),
+                    ", ".join(f"{index.name} (table size {index.table_size})" for index in found),
+                )
+                lock_row: Final = conn.execute(
+                    "SELECT pg_try_advisory_lock(%s)", (INDEX_REPAIR_ADVISORY_LOCK_KEY,)
+                ).fetchone()
+                if lock_row is None or not lock_row[0]:
+                    logger.info("Another replica is already rebuilding the invalid indexes, skipping")
+                    return False
+                for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema):
+                    ProxyExtrasDBManager._repair_index(conn, index)
+                remaining: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
+        except psycopg.Error as e:
+            logger.warning("Could not check for invalid indexes, will retry on the next startup. Error: %s", e)
+            return False
+        return not remaining
+
     @staticmethod
     def _setup_database_v2(use_migrate: bool) -> bool:
         """
@@ -886,6 +999,7 @@ class ProxyExtrasDBManager:
             use_migrate=use_migrate, use_v2_resolver=use_v2_resolver
         )
         if migrated:
+            ProxyExtrasDBManager.repair_invalid_indexes()
             ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
         return migrated
 
diff --git a/tests/proxy_migration_tests/test_invalid_index_repair.py b/tests/proxy_migration_tests/test_invalid_index_repair.py
new file mode 100644
index 00000000000..787a62b361b
--- /dev/null
+++ b/tests/proxy_migration_tests/test_invalid_index_repair.py
@@ -0,0 +1,238 @@
+import os
+import subprocess
+import threading
+import uuid
+from collections.abc import Iterator, Mapping
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import pytest
+from litellm_proxy_extras.utils import INDEX_REPAIR_ADVISORY_LOCK_KEY, ProxyExtrasDBManager
+
+psycopg = pytest.importorskip("psycopg")
+
+pytestmark = pytest.mark.timeout(120)
+
+requires_db: Final = pytest.mark.skipif(
+    "DATABASE_URL" not in os.environ,
+    reason="requires a postgres database (DATABASE_URL)",
+)
+
+HEALTH_TABLE: Final = "LiteLLM_HealthCheckTable"
+HEALTH_INDEX: Final = "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx"
+HEALTH_INDEX_COLUMNS: Final = '"model_id", "model_name", "checked_at" DESC'
+LOOKALIKE_TABLE: Final = "LiteLLMLookalikeTable"
+LOOKALIKE_INDEX: Final = "LiteLLMLookalikeTable_id_idx"
+PARTITIONED_TABLE: Final = "LiteLLM_PartitionedTable"
+PARTITIONED_INDEX: Final = "LiteLLM_PartitionedTable_id_idx"
+
+
+def _base_url() -> str:
+    return os.environ["DATABASE_URL"].split("?")[0]
+
+
+def _index_validity(schema: str) -> Mapping[str, bool]:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        rows = conn.execute(
+            "SELECT c.relname, i.indisvalid FROM pg_index i "
+            "JOIN pg_class c ON c.oid = i.indexrelid "
+            "JOIN pg_namespace n ON n.oid = c.relnamespace "
+            "WHERE n.nspname = %s",
+            (schema,),
+        ).fetchall()
+    return MappingProxyType(dict(rows))
+
+
+def _interrupt_concurrent_build(schema: str, table: str, statement: str) -> None:
+    """Abort a CONCURRENTLY build while it waits on an older snapshot, the same
+    spot the deadlock loser dies at, so it leaves its index INVALID."""
+    with psycopg.connect(_base_url()) as pin:
+        pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        pin.execute(f'SELECT count(*) FROM "{schema}"."{table}"')
+        with psycopg.connect(_base_url(), autocommit=True) as builder:
+            builder.execute("SET statement_timeout = '1s'")
+            with pytest.raises(psycopg.errors.QueryCanceled):
+                builder.execute(statement)
+
+
+def _leave_invalid_index(schema: str, table: str, index: str, columns: str) -> None:
+    _interrupt_concurrent_build(
+        schema, table, f'CREATE INDEX CONCURRENTLY "{index}" ON "{schema}"."{table}" ({columns})'
+    )
+
+
+def _leave_invalid_reindex_leftover(schema: str, table: str, index: str) -> None:
+    _interrupt_concurrent_build(schema, table, f'REINDEX INDEX CONCURRENTLY "{schema}"."{index}"')
+
+
+@pytest.fixture
+def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    schema: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE SCHEMA "{schema}"')
+        conn.execute(
+            f'CREATE TABLE "{schema}"."{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)'
+        )
+        conn.execute(f'CREATE TABLE "{schema}"."{LOOKALIKE_TABLE}" (id TEXT)')
+
+    monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={schema}")
+    yield schema
+
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
+
+
+@requires_db
+def test_repair_rebuilds_invalid_litellm_indexes_and_leaves_lookalike_tables_alone(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    _leave_invalid_index(scratch_schema, LOOKALIKE_TABLE, LOOKALIKE_INDEX, "id")
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: False, LOOKALIKE_INDEX: False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True, LOOKALIKE_INDEX: False}
+
+
+@requires_db
+def test_repair_drops_leftovers_of_interrupted_rebuilds(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    _leave_invalid_reindex_leftover(scratch_schema, HEALTH_TABLE, HEALTH_INDEX)
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, f"{HEALTH_TABLE}_model_id_idx_ccold", '"model_id"')
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, f"{HEALTH_TABLE}_model_id_idx_ccnew1", '"model_id"')
+    before: Final = _index_validity(scratch_schema)
+    assert len(before) == 4
+    assert set(before.values()) == {False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_is_a_no_op_when_every_index_is_valid(scratch_schema: str) -> None:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE INDEX "{HEALTH_INDEX}" ON "{scratch_schema}"."{HEALTH_TABLE}" ({HEALTH_INDEX_COLUMNS})')
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_leaves_partitioned_parent_indexes_alone(scratch_schema: str) -> None:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE TABLE "{scratch_schema}"."{PARTITIONED_TABLE}" (id INT) PARTITION BY RANGE (id)')
+        conn.execute(
+            f'CREATE TABLE "{scratch_schema}"."{PARTITIONED_TABLE}_p0" '
+            f'PARTITION OF "{scratch_schema}"."{PARTITIONED_TABLE}" FOR VALUES FROM (0) TO (10)'
+        )
+        conn.execute(f'CREATE INDEX "{PARTITIONED_INDEX}" ON ONLY "{scratch_schema}"."{PARTITIONED_TABLE}" (id)')
+    assert _index_validity(scratch_schema) == {PARTITIONED_INDEX: False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {PARTITIONED_INDEX: False}
+
+
+@requires_db
+def test_repair_yields_to_the_replica_holding_the_repair_lock(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+
+    with psycopg.connect(_base_url(), autocommit=True) as other_replica:
+        other_replica.execute("SELECT pg_advisory_lock(%s)", (INDEX_REPAIR_ADVISORY_LOCK_KEY,))
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is False
+        assert _index_validity(scratch_schema) == {HEALTH_INDEX: False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_gives_up_on_a_blocked_rebuild_and_finishes_it_on_the_next_startup(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+
+    with psycopg.connect(_base_url()) as pin:
+        pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        pin.execute(f'SELECT count(*) FROM "{scratch_schema}"."{HEALTH_TABLE}"')
+        assert ProxyExtrasDBManager.repair_invalid_indexes(lock_timeout="1s") is False
+        blocked: Final = _index_validity(scratch_schema)
+        assert blocked[HEALTH_INDEX] is False
+        assert [name for name in blocked if name.endswith("_ccnew")]
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+def _hold_snapshot(schema: str, table: str, pinned: threading.Event, seconds: float) -> None:
+    with psycopg.connect(_base_url()) as pin:
+        pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        pin.execute(f'SELECT count(*) FROM "{schema}"."{table}"')
+        pinned.set()
+        pin.execute("SELECT pg_sleep(%s)", (seconds,))
+
+
+@requires_db
+def test_repair_outlives_a_statement_timeout_passed_through_database_url_options(
+    scratch_schema: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={scratch_schema}&options=-c%20statement_timeout%3D2000")
+    pinned: Final = threading.Event()
+    holder: Final = threading.Thread(target=_hold_snapshot, args=(scratch_schema, HEALTH_TABLE, pinned, 5.0))
+    holder.start()
+    pinned.wait()
+    try:
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+    finally:
+        holder.join()
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_defaults_to_the_public_schema(monkeypatch: pytest.MonkeyPatch) -> None:
+    table: Final = f"LiteLLM_ScratchTable_{uuid.uuid4().hex[:8]}"
+    index: Final = f"{table}_id_idx"
+    monkeypatch.setenv("DATABASE_URL", _base_url())
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE TABLE public."{table}" (id TEXT)')
+    try:
+        _leave_invalid_index("public", table, index, "id")
+        assert _index_validity("public")[index] is False
+
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+        assert _index_validity("public")[index] is True
+    finally:
+        with psycopg.connect(_base_url(), autocommit=True) as conn:
+            conn.execute(f'DROP TABLE public."{table}"')
+
+
+def test_repair_survives_an_unreachable_database(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is False
+
+
+class _MigrateDeployApplied:
+    stdout = "Applied migration.\n"
+    stderr = ""
+    returncode = 0
+
+
+@requires_db
+@pytest.mark.parametrize("use_v2_resolver", [True, False])
+def test_setup_database_repairs_the_index_after_a_recovered_deploy(
+    scratch_schema: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, use_v2_resolver: bool
+) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    (tmp_path / "schema.prisma").write_text("// stub")
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
+    monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", lambda *_, **__: True)
+    monkeypatch.setattr(subprocess, "run", lambda *_, **__: _MigrateDeployApplied())
+
+    assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}

From 2d64020d432f1c6112b836a752720a25a5ab93e9 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 15:24:42 -0700
Subject: [PATCH 02/27] fix(proxy-extras): run the index repair over DIRECT_URL
 and exercise it through a real migrate deploy

---
 .../litellm_proxy_extras/utils.py             |  7 ++-
 .../test_invalid_index_repair.py              | 63 ++++++++++++-------
 2 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py
index df019d78078..75fda59f28e 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/utils.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py
@@ -721,8 +721,11 @@ class ProxyExtrasDBManager:
         INVALID (a migration deadlock between replicas is the usual cause; the
         retried migration skips them because of IF NOT EXISTS). Never raises:
         returns True when no invalid index remains, False when the repair was
-        skipped or failed and will be retried on the next startup."""
-        database_url: Final = os.getenv("DATABASE_URL")
+        skipped or failed and will be retried on the next startup. Runs over
+        DIRECT_URL when set: the session settings, the advisory lock and REINDEX
+        CONCURRENTLY all need one server session, which a transaction pooler
+        does not give."""
+        database_url: Final = os.getenv("DIRECT_URL") or os.getenv("DATABASE_URL")
         if not database_url:
             return False
 
diff --git a/tests/proxy_migration_tests/test_invalid_index_repair.py b/tests/proxy_migration_tests/test_invalid_index_repair.py
index 787a62b361b..2e46775cd2b 100644
--- a/tests/proxy_migration_tests/test_invalid_index_repair.py
+++ b/tests/proxy_migration_tests/test_invalid_index_repair.py
@@ -1,9 +1,7 @@
 import os
-import subprocess
 import threading
 import uuid
 from collections.abc import Iterator, Mapping
-from pathlib import Path
 from types import MappingProxyType
 from typing import Final
 
@@ -66,16 +64,14 @@ def _leave_invalid_reindex_leftover(schema: str, table: str, index: str) -> None
     _interrupt_concurrent_build(schema, table, f'REINDEX INDEX CONCURRENTLY "{schema}"."{index}"')
 
 
-@pytest.fixture
-def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+def _scratch_schema(monkeypatch: pytest.MonkeyPatch, *table_definitions: str) -> Iterator[str]:
     schema: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
     with psycopg.connect(_base_url(), autocommit=True) as conn:
         conn.execute(f'CREATE SCHEMA "{schema}"')
-        conn.execute(
-            f'CREATE TABLE "{schema}"."{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)'
-        )
-        conn.execute(f'CREATE TABLE "{schema}"."{LOOKALIKE_TABLE}" (id TEXT)')
+        for definition in table_definitions:
+            conn.execute(f'CREATE TABLE "{schema}".{definition}')
 
+    monkeypatch.delenv("DIRECT_URL", raising=False)
     monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={schema}")
     yield schema
 
@@ -83,6 +79,20 @@ def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
         conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
 
 
+@pytest.fixture
+def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    yield from _scratch_schema(
+        monkeypatch,
+        f'"{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)',
+        f'"{LOOKALIKE_TABLE}" (id TEXT)',
+    )
+
+
+@pytest.fixture
+def empty_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    yield from _scratch_schema(monkeypatch)
+
+
 @requires_db
 def test_repair_rebuilds_invalid_litellm_indexes_and_leaves_lookalike_tables_alone(scratch_schema: str) -> None:
     _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
@@ -210,29 +220,38 @@ def test_repair_defaults_to_the_public_schema(monkeypatch: pytest.MonkeyPatch) -
 
 
 def test_repair_survives_an_unreachable_database(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("DIRECT_URL", raising=False)
     monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
 
     assert ProxyExtrasDBManager.repair_invalid_indexes() is False
 
 
-class _MigrateDeployApplied:
-    stdout = "Applied migration.\n"
-    stderr = ""
-    returncode = 0
+@requires_db
+def test_repair_runs_over_direct_url_when_set(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    direct_url: Final = os.environ["DATABASE_URL"]
+    with pytest.MonkeyPatch.context() as env:
+        env.setenv("DIRECT_URL", direct_url)
+        env.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+def _invalidate_deployed_index(schema: str) -> None:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'DROP INDEX "{schema}"."{HEALTH_INDEX}"')
+    _leave_invalid_index(schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
 
 
 @requires_db
+@pytest.mark.timeout(300)
 @pytest.mark.parametrize("use_v2_resolver", [True, False])
-def test_setup_database_repairs_the_index_after_a_recovered_deploy(
-    scratch_schema: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, use_v2_resolver: bool
-) -> None:
-    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
-    (tmp_path / "schema.prisma").write_text("// stub")
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
-    monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", lambda *_, **__: True)
-    monkeypatch.setattr(subprocess, "run", lambda *_, **__: _MigrateDeployApplied())
+def test_setup_database_repairs_the_index_after_a_recovered_deploy(empty_schema: str, use_v2_resolver: bool) -> None:
+    assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
+    _invalidate_deployed_index(empty_schema)
+    assert _index_validity(empty_schema)[HEALTH_INDEX] is False
 
     assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
 
-    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+    assert _index_validity(empty_schema)[HEALTH_INDEX] is True

From bfdaedf51b13f4bc8872d51a99d7f288fc0cc56f Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 16:23:50 -0700
Subject: [PATCH 03/27] fix(proxy-extras): take the repair schema from
 DATABASE_URL and deploy the real-migration test on a fresh database

---
 .../litellm_proxy_extras/utils.py             | 17 +++----
 .../test_invalid_index_repair.py              | 47 +++++++++++--------
 2 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py
index 75fda59f28e..40bc4cd1dfd 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/utils.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py
@@ -721,12 +721,13 @@ class ProxyExtrasDBManager:
         INVALID (a migration deadlock between replicas is the usual cause; the
         retried migration skips them because of IF NOT EXISTS). Never raises:
         returns True when no invalid index remains, False when the repair was
-        skipped or failed and will be retried on the next startup. Runs over
-        DIRECT_URL when set: the session settings, the advisory lock and REINDEX
-        CONCURRENTLY all need one server session, which a transaction pooler
-        does not give."""
-        database_url: Final = os.getenv("DIRECT_URL") or os.getenv("DATABASE_URL")
-        if not database_url:
+        skipped or failed and will be retried on the next startup. Looks in the
+        schema DATABASE_URL names, the only URL Prisma migrates through, but
+        connects over DIRECT_URL when set: the session settings, the advisory
+        lock and REINDEX CONCURRENTLY all need one server session, which a
+        transaction pooler does not give."""
+        prisma_url: Final = os.getenv("DATABASE_URL")
+        if not prisma_url:
             return False
 
         try:
@@ -739,8 +740,8 @@ class ProxyExtrasDBManager:
             )
             return False
 
-        schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
-        cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        schema: Final = ProxyExtrasDBManager._prisma_schema_param(prisma_url) or "public"
+        cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(os.getenv("DIRECT_URL") or prisma_url)
         try:
             with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
                 conn.execute("SET statement_timeout = 0")
diff --git a/tests/proxy_migration_tests/test_invalid_index_repair.py b/tests/proxy_migration_tests/test_invalid_index_repair.py
index 2e46775cd2b..741fa7386df 100644
--- a/tests/proxy_migration_tests/test_invalid_index_repair.py
+++ b/tests/proxy_migration_tests/test_invalid_index_repair.py
@@ -64,12 +64,15 @@ def _leave_invalid_reindex_leftover(schema: str, table: str, index: str) -> None
     _interrupt_concurrent_build(schema, table, f'REINDEX INDEX CONCURRENTLY "{schema}"."{index}"')
 
 
-def _scratch_schema(monkeypatch: pytest.MonkeyPatch, *table_definitions: str) -> Iterator[str]:
+@pytest.fixture
+def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
     schema: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
     with psycopg.connect(_base_url(), autocommit=True) as conn:
         conn.execute(f'CREATE SCHEMA "{schema}"')
-        for definition in table_definitions:
-            conn.execute(f'CREATE TABLE "{schema}".{definition}')
+        conn.execute(
+            f'CREATE TABLE "{schema}"."{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)'
+        )
+        conn.execute(f'CREATE TABLE "{schema}"."{LOOKALIKE_TABLE}" (id TEXT)')
 
     monkeypatch.delenv("DIRECT_URL", raising=False)
     monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={schema}")
@@ -80,17 +83,22 @@ def _scratch_schema(monkeypatch: pytest.MonkeyPatch, *table_definitions: str) ->
 
 
 @pytest.fixture
-def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
-    yield from _scratch_schema(
-        monkeypatch,
-        f'"{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)',
-        f'"{LOOKALIKE_TABLE}" (id TEXT)',
-    )
+def fresh_database(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    """A brand-new database, what a first deploy sees. A scratch schema would
+    not do: the migrations guard on pg_constraint by name across every schema,
+    so a LiteLLM schema already pushed into public makes them skip and then
+    fail, which is exactly what CI's database looks like."""
+    admin_url: Final = _base_url()
+    name: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
+    with psycopg.connect(admin_url, autocommit=True) as conn:
+        conn.execute(f'CREATE DATABASE "{name}"')
 
+    monkeypatch.delenv("DIRECT_URL", raising=False)
+    monkeypatch.setenv("DATABASE_URL", f"{admin_url.rsplit('/', 1)[0]}/{name}")
+    yield "public"
 
-@pytest.fixture
-def empty_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
-    yield from _scratch_schema(monkeypatch)
+    with psycopg.connect(admin_url, autocommit=True) as conn:
+        conn.execute(f'DROP DATABASE "{name}" WITH (FORCE)')
 
 
 @requires_db
@@ -227,12 +235,11 @@ def test_repair_survives_an_unreachable_database(monkeypatch: pytest.MonkeyPatch
 
 
 @requires_db
-def test_repair_runs_over_direct_url_when_set(scratch_schema: str) -> None:
+def test_repair_connects_over_direct_url_but_looks_in_the_schema_database_url_names(scratch_schema: str) -> None:
     _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
-    direct_url: Final = os.environ["DATABASE_URL"]
     with pytest.MonkeyPatch.context() as env:
-        env.setenv("DIRECT_URL", direct_url)
-        env.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
+        env.setenv("DIRECT_URL", f"{_base_url()}?schema=public")
+        env.setenv("DATABASE_URL", f"postgresql://u:p@127.0.0.1:9/x?schema={scratch_schema}")
         assert ProxyExtrasDBManager.repair_invalid_indexes() is True
 
     assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
@@ -247,11 +254,11 @@ def _invalidate_deployed_index(schema: str) -> None:
 @requires_db
 @pytest.mark.timeout(300)
 @pytest.mark.parametrize("use_v2_resolver", [True, False])
-def test_setup_database_repairs_the_index_after_a_recovered_deploy(empty_schema: str, use_v2_resolver: bool) -> None:
+def test_setup_database_repairs_the_index_after_a_recovered_deploy(fresh_database: str, use_v2_resolver: bool) -> None:
     assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
-    _invalidate_deployed_index(empty_schema)
-    assert _index_validity(empty_schema)[HEALTH_INDEX] is False
+    _invalidate_deployed_index(fresh_database)
+    assert _index_validity(fresh_database)[HEALTH_INDEX] is False
 
     assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
 
-    assert _index_validity(empty_schema)[HEALTH_INDEX] is True
+    assert _index_validity(fresh_database)[HEALTH_INDEX] is True

From 48cc4efca38b46e1f561da4728954d06bbe223c7 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 20:37:17 -0700
Subject: [PATCH 04/27] fix(least-busy): share in-flight request counts across
 proxy workers

Least-busy kept one dict of in-flight counts per model group in the
router cache, which reads in-memory first, so every worker and replica
routed on its own stale copy and each write overwrote the shared value.

Counts now live in one key per deployment, incremented and read through
Redis when the router has a Redis cache, and in the process-local cache
otherwise.
---
 basedpyright-code-budget.json                 |  12 +-
 litellm/caching/redis_cache.py                |   4 +-
 litellm/router_strategy/least_busy.py         | 372 +++++++++---------
 ruff-strict-budget.json                       |  10 +-
 .../local_testing/test_least_busy_routing.py  |  46 ++-
 .../router_strategy/test_least_busy.py        | 151 +++++++
 type-discipline-budget.json                   |   8 +-
 7 files changed, 376 insertions(+), 227 deletions(-)
 create mode 100644 tests/test_litellm/router_strategy/test_least_busy.py

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 0b0a61192e6..c21dc76743a 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5570
+    "limit": 5551
   },
   "reportMissingTypeArgument": {
-    "limit": 15281
+    "limit": 15277
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -99,19 +99,19 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44358
+    "limit": 44357
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38271
+    "limit": 38240
   },
   "reportUnknownParameterType": {
-    "limit": 19584
+    "limit": 19558
   },
   "reportUnknownVariableType": {
-    "limit": 29814
+    "limit": 29781
   },
   "reportUnnecessaryCast": {
     "limit": 110
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index aaee7188d86..d4c47914394 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -680,7 +680,7 @@ class RedisCache(BaseCache):
             # NON blocking - notify users Redis is throwing an exception
             print_verbose(f"litellm.caching.caching: set() - Got exception from REDIS : {e}")
 
-    def increment_cache(self, key, value: int, ttl: float | None = None, **kwargs) -> int:
+    def increment_cache(self, key, value: int, ttl: float | None = None, refresh_ttl: bool = False, **kwargs) -> int:
         _redis_client: Final = self.redis_client
         start_time = time.time()
         set_ttl: Final = self.get_ttl(ttl=ttl)
@@ -701,7 +701,7 @@ class RedisCache(BaseCache):
             if set_ttl is not None:
                 # check if key already has ttl, if not -> set ttl
                 start_time = time.time()
-                current_ttl: Final = _redis_client.ttl(key)
+                current_ttl: Final = -1 if refresh_ttl else _redis_client.ttl(key)
                 end_time = time.time()
                 _duration = end_time - start_time
                 self.service_logger_obj.service_success_hook(
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index 1433e8ba4d4..de6a4c4f59a 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -1,17 +1,96 @@
-#### What this does ####
-#   identifies least busy deployment
-#   How is this achieved?
-#   - Before each call, have the router print the state of requests {"deployment": "requests_in_flight"}
-#   - use litellm.input_callbacks to log when a request is just about to be made to a model - {"deployment-id": traffic}
-#   - use litellm.success + failure callbacks to log when a request completed
-#   - in get_available_deployment, for a given model group name -> pick based on traffic
-
-import random
+from collections.abc import Mapping, Sequence
 from typing import Final
 
+from pydantic import TypeAdapter, ValidationError
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm._logging import verbose_router_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
 
+IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60
+
+
+class _ModelInfo(TypedDict, total=False):
+    id: ReadOnly[str | int | None]
+
+
+class _Metadata(TypedDict, total=False):
+    model_group: ReadOnly[str | None]
+
+
+class _LitellmParams(TypedDict, total=False):
+    metadata: ReadOnly[_Metadata | None]
+    model_info: ReadOnly[_ModelInfo | None]
+
+
+class _CallKwargs(TypedDict, total=False):
+    litellm_params: ReadOnly[_LitellmParams | None]
+
+
+class _DeploymentModelInfo(TypedDict):
+    id: ReadOnly[str | int]
+
+
+class _Deployment(TypedDict):
+    model_info: ReadOnly[_DeploymentModelInfo]
+
+
+_CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
+_DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
+_REDIS_COUNTS: Final = TypeAdapter(dict[str, float | None])
+_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...])
+
+
+def _request_count_key(model_group: str, deployment_id: str) -> str:
+    return f"{model_group}_request_count:{deployment_id}"
+
+
+def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
+    try:
+        call: Final = _CALL_KWARGS.validate_python(kwargs)
+    except ValidationError:
+        return None
+    litellm_params: Final = call.get("litellm_params")
+    metadata: Final = litellm_params.get("metadata") if litellm_params else None
+    model_info: Final = litellm_params.get("model_info") if litellm_params else None
+    model_group: Final = metadata.get("model_group") if metadata else None
+    deployment_id: Final = model_info.get("id") if model_info else None
+    if model_group is None or deployment_id is None:
+        return None
+    return model_group, str(deployment_id)
+
+
+def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]:
+    return tuple(
+        _request_count_key(model_group, str(deployment["model_info"]["id"]))
+        for deployment in _DEPLOYMENTS.validate_python(healthy_deployments)
+    )
+
+
+def _as_count(value: float | None) -> int:
+    return 0 if value is None else int(value)
+
+
+def _least_busy(
+    healthy_deployments: Sequence[Mapping[str, object]], counts: tuple[int, ...]
+) -> Mapping[str, object] | None:
+    if not healthy_deployments:
+        return None
+    return healthy_deployments[min(range(len(healthy_deployments)), key=lambda index: counts[index])]
+
+
+def _warn_unreadable(model_group: str, error: Exception) -> None:
+    verbose_router_logger.warning(
+        "least-busy routing could not read the in-flight counts for %s, treating every deployment as idle: %s",
+        model_group,
+        error,
+    )
+
+
+def _warn_unwritable(key: str, error: Exception) -> None:
+    verbose_router_logger.warning("least-busy routing could not update the in-flight count under %s: %s", key, error)
+
 
 class LeastBusyLoggingHandler(CustomLogger):
     test_flag: bool = False
@@ -21,194 +100,101 @@ class LeastBusyLoggingHandler(CustomLogger):
     def __init__(self, router_cache: DualCache):
         self.router_cache = router_cache
 
-    def log_pre_api_call(self, model, messages, kwargs):
-        """
-        Log when a model is being used.
+    def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
+        self._increment(kwargs, 1)
 
-        Caching based on model group.
-        """
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
+    def log_success_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        self._increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_success += 1
 
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # update cache
-                request_count_dict: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-                request_count_dict[id] = request_count_dict.get(id, 0) + 1
+    def log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        self._increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_failure += 1
 
-                self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
-        except Exception:
-            pass
+    async def async_log_success_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        await self._async_increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_success += 1
 
-    def log_success_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_success += 1
-        except Exception:
-            pass
-
-    def log_failure_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_failure += 1
-        except Exception:
-            pass
-
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = await self.router_cache.async_get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_success += 1
-        except Exception:
-            pass
-
-    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = await self.router_cache.async_get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_failure += 1
-        except Exception:
-            pass
-
-    def _get_available_deployments(
-        self,
-        healthy_deployments: list,
-        all_deployments: dict,
-    ):
-        """
-        Helper to get deployments using least busy strategy
-        """
-        for d in healthy_deployments:
-            ## if healthy deployment not yet used
-            if d["model_info"]["id"] not in all_deployments:
-                all_deployments[d["model_info"]["id"]] = 0
-        # map deployment to id
-        # pick least busy deployment
-        min_traffic = float("inf")
-        min_deployment = None
-        for k, v in all_deployments.items():
-            if v < min_traffic:
-                min_traffic = v
-                min_deployment = k
-        if min_deployment is not None:
-            ## check if min deployment is a string, if so, cast it to int
-            for m in healthy_deployments:
-                if m["model_info"]["id"] == min_deployment:
-                    return m
-            min_deployment = random.choice(healthy_deployments)
-        else:
-            min_deployment = random.choice(healthy_deployments)
-        return min_deployment
+    async def async_log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        await self._async_increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_failure += 1
 
     def get_available_deployments(
-        self,
-        model_group: str,
-        healthy_deployments: list,
-    ):
-        """
-        Sync helper to get deployments using least busy strategy
-        """
-        request_count_api_key: Final = f"{model_group}_request_count"
-        all_deployments: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-        return self._get_available_deployments(
-            healthy_deployments=healthy_deployments,
-            all_deployments=all_deployments,
-        )
+        self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
+    ) -> Mapping[str, object] | None:
+        keys: Final = _request_count_keys(model_group, healthy_deployments)
+        try:
+            counts: Final = tuple(_as_count(value) for value in self._read_counts(keys))
+        except Exception as e:
+            _warn_unreadable(model_group, e)
+            return _least_busy(healthy_deployments, (0,) * len(keys))
+        return _least_busy(healthy_deployments, counts)
 
-    async def async_get_available_deployments(self, model_group: str, healthy_deployments: list):
-        """
-        Async helper to get deployments using least busy strategy
-        """
-        request_count_api_key: Final = f"{model_group}_request_count"
-        all_deployments: Final = await self.router_cache.async_get_cache(key=request_count_api_key) or {}
-        return self._get_available_deployments(
-            healthy_deployments=healthy_deployments,
-            all_deployments=all_deployments,
-        )
+    async def async_get_available_deployments(
+        self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
+    ) -> Mapping[str, object] | None:
+        keys: Final = _request_count_keys(model_group, healthy_deployments)
+        try:
+            counts: Final = tuple(_as_count(value) for value in await self._async_read_counts(keys))
+        except Exception as e:
+            _warn_unreadable(model_group, e)
+            return _least_busy(healthy_deployments, (0,) * len(keys))
+        return _least_busy(healthy_deployments, counts)
+
+    def _increment(self, kwargs: Mapping[str, object], delta: int) -> None:
+        ref: Final = _deployment_ref(kwargs)
+        if ref is None:
+            return
+        key: Final = _request_count_key(*ref)
+        redis_cache: Final = self.router_cache.redis_cache
+        try:
+            if redis_cache is None:
+                self.router_cache.increment_cache(key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
+            else:
+                redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+        except Exception as e:
+            _warn_unwritable(key, e)
+
+    async def _async_increment(self, kwargs: Mapping[str, object], delta: int) -> None:
+        ref: Final = _deployment_ref(kwargs)
+        if ref is None:
+            return
+        key: Final = _request_count_key(*ref)
+        redis_cache: Final = self.router_cache.redis_cache
+        try:
+            if redis_cache is None:
+                await self.router_cache.async_increment_cache(
+                    key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
+                )
+            else:
+                await redis_cache.async_increment(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+        except Exception as e:
+            _warn_unwritable(key, e)
+
+    def _read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is None:
+            return _MEMORY_COUNTS.validate_python(self.router_cache.batch_get_cache(list(keys), local_only=True))
+        by_key: Final = _REDIS_COUNTS.validate_python(redis_cache.batch_get_cache(key_list=list(keys)))
+        return tuple(by_key.get(key) for key in keys)
+
+    async def _async_read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is None:
+            return _MEMORY_COUNTS.validate_python(
+                await self.router_cache.async_batch_get_cache(list(keys), local_only=True)
+            )
+        by_key: Final = _REDIS_COUNTS.validate_python(await redis_cache.async_batch_get_cache(key_list=list(keys)))
+        return tuple(by_key.get(key) for key in keys)
diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index fd7b30bc314..ceff8c6e15c 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -1,6 +1,6 @@
 {
   "ANN001": {
-    "limit": 2956
+    "limit": 2937
   },
   "ANN002": {
     "limit": 71
@@ -9,10 +9,10 @@
     "limit": 806
   },
   "ANN201": {
-    "limit": 1979
+    "limit": 1972
   },
   "ANN202": {
-    "limit": 831
+    "limit": 830
   },
   "ANN204": {
     "limit": 683
@@ -57,7 +57,7 @@
     "limit": 3
   },
   "BLE001": {
-    "limit": 2916
+    "limit": 2915
   },
   "C401": {
     "limit": 8
@@ -189,7 +189,7 @@
     "limit": 0
   },
   "S110": {
-    "limit": 217
+    "limit": 212
   },
   "S112": {
     "limit": 22
diff --git a/tests/local_testing/test_least_busy_routing.py b/tests/local_testing/test_least_busy_routing.py
index 18ab8bf779d..fb83d4e601f 100644
--- a/tests/local_testing/test_least_busy_routing.py
+++ b/tests/local_testing/test_least_busy_routing.py
@@ -33,8 +33,8 @@ def test_model_added():
         }
     }
     least_busy_logger.log_pre_api_call(model="test", messages=[], kwargs=kwargs)
-    request_count_api_key = f"gpt-3.5-turbo_request_count"
-    assert test_cache.get_cache(key=request_count_api_key) is not None
+    request_count_api_key = "gpt-3.5-turbo_request_count:1234"
+    assert test_cache.get_cache(key=request_count_api_key) == 1
 
 
 def test_get_available_deployments():
@@ -52,8 +52,8 @@ def test_get_available_deployments():
         }
     }
     least_busy_logger.log_pre_api_call(model="test", messages=[], kwargs=kwargs)
-    request_count_api_key = f"{model_group}_request_count"
-    assert test_cache.get_cache(key=request_count_api_key) is not None
+    request_count_api_key = f"{model_group}_request_count:1234"
+    assert test_cache.get_cache(key=request_count_api_key) == 1
 
 
 # test_get_available_deployments()
@@ -104,15 +104,20 @@ async def test_router_get_available_deployments(async_test):
     router.leastbusy_logger.test_flag = True
 
     model_group = "azure-model"
-    request_count_dict = {1: 10, 2: 54, 3: 100}
-    cache_key = f"{model_group}_request_count"
+    request_count_dict = {"1": 10, "2": 54, "3": 100}
+    cache_keys = {
+        deployment_id: f"{model_group}_request_count:{deployment_id}"
+        for deployment_id in request_count_dict
+    }
     if async_test is True:
-        await router.cache.async_set_cache(key=cache_key, value=request_count_dict)
+        for deployment_id, count in request_count_dict.items():
+            await router.cache.async_set_cache(key=cache_keys[deployment_id], value=count)
         deployment = await router.async_get_available_deployment(
             model=model_group, messages=None, request_kwargs={}
         )
     else:
-        router.cache.set_cache(key=cache_key, value=request_count_dict)
+        for deployment_id, count in request_count_dict.items():
+            router.cache.set_cache(key=cache_keys[deployment_id], value=count)
         deployment = router.get_available_deployment(model=model_group, messages=None)
     print(f"deployment: {deployment}")
     assert deployment["model_info"]["id"] == "1"
@@ -124,15 +129,18 @@ async def test_router_get_available_deployments(async_test):
         messages=[{"role": "user", "content": "Hey, how's it going?"}],
     )
 
-    return_dict = router.cache.get_cache(key=cache_key)
-
     # wait 2 seconds
     time.sleep(2)
 
+    return_dict = {
+        deployment_id: router.cache.get_cache(key=cache_key)
+        for deployment_id, cache_key in cache_keys.items()
+    }
+
     assert router.leastbusy_logger.logged_success == 1
-    assert return_dict[1] == 10
-    assert return_dict[2] == 54
-    assert return_dict[3] == 100
+    assert return_dict["1"] == 10
+    assert return_dict["2"] == 54
+    assert return_dict["3"] == 100
 
 
 ## Test with Real calls ##
@@ -192,9 +200,11 @@ async def test_router_atext_completion_streaming():
     await asyncio.sleep(random.uniform(0, 2))
     await router.atext_completion(model=model, prompt=prompt, stream=True)
 
-    cache_key = f"{model}_request_count"
     ## check if calls equally distributed
-    cache_dict = router.cache.get_cache(key=cache_key)
+    cache_dict = {
+        deployment_id: router.cache.get_cache(key=f"{model}_request_count:{deployment_id}")
+        for deployment_id in ("1", "2", "3")
+    }
     for k, v in cache_dict.items():
         assert v == 1, f"Failed. K={k} called v={v} times, cache_dict={cache_dict}"
 
@@ -259,8 +269,10 @@ async def test_router_completion_streaming():
     await asyncio.sleep(random.uniform(0, 2))
     await router.acompletion(model=model, messages=messages, stream=True)
 
-    cache_key = f"{model}_request_count"
     ## check if calls equally distributed
-    cache_dict = router.cache.get_cache(key=cache_key)
+    cache_dict = {
+        deployment_id: router.cache.get_cache(key=f"{model}_request_count:{deployment_id}")
+        for deployment_id in ("1", "2", "3")
+    }
     for k, v in cache_dict.items():
         assert v == 1, f"Failed. K={k} called v={v} times, cache_dict={cache_dict}"
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
new file mode 100644
index 00000000000..32a226a5d5b
--- /dev/null
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -0,0 +1,151 @@
+import json
+from typing import Final
+
+import pytest
+
+from litellm.caching.caching import DualCache
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.router_strategy.least_busy import IN_FLIGHT_COUNT_TTL_SECONDS, LeastBusyLoggingHandler
+
+GROUP: Final = "least-busy-group"
+DEPLOYMENT_A: Final[dict[str, object]] = {"model_info": {"id": "dep-a"}}
+DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}}
+HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B]
+
+
+def _call_kwargs(deployment_id: str) -> dict[str, object]:
+    return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}}
+
+
+class SharedRedisCounters:
+    """Stores JSON strings and hands back a fresh object per read, the way a real Redis client does."""
+
+    def __init__(self) -> None:
+        self.encoded: dict[str, str] = {}
+        self.ttls: dict[str, float] = {}
+
+    def count(self, key: str) -> object:
+        raw: Final = self.encoded.get(key)
+        return None if raw is None else json.loads(raw)
+
+    def get_cache(self, key: str, **kwargs: object) -> object:
+        return self.count(key)
+
+    def set_cache(self, key: str, value: object, **kwargs: object) -> None:
+        self.encoded[key] = json.dumps(value)
+
+    async def async_get_cache(self, key: str, **kwargs: object) -> object:
+        return self.count(key)
+
+    async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
+        self.set_cache(key, value)
+
+    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+        current: Final = self.count(key) or 0
+        assert isinstance(current, int)
+        incremented: Final = current + value
+        self.encoded[key] = json.dumps(incremented)
+        if ttl is not None and (refresh_ttl or key not in self.ttls):
+            self.ttls[key] = ttl
+        return incremented
+
+    async def async_increment(self, key: str, value: float, ttl: int | None = None, refresh_ttl: bool = False) -> float:
+        return self.increment_cache(key, int(value), ttl, refresh_ttl)
+
+    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+        return {key: self.count(key) for key in key_list}
+
+    async def async_batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+        return self.batch_get_cache(key_list)
+
+
+def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler:
+    cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared)  # pyright: ignore[reportArgumentType]  # duck-typed Redis double
+    return LeastBusyLoggingHandler(router_cache=cache)
+
+
+@pytest.mark.asyncio
+async def test_worker_routes_around_a_request_another_worker_started() -> None:
+    shared: Final = SharedRedisCounters()
+    streaming_worker: Final = _worker(shared)
+    picking_worker: Final = _worker(shared)
+
+    picking_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    await picking_worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    streaming_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    await streaming_worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+
+
+def test_sync_pick_reads_the_shared_counts() -> None:
+    shared: Final = SharedRedisCounters()
+    streaming_worker: Final = _worker(shared)
+    picking_worker: Final = _worker(shared)
+
+    picking_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    picking_worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    streaming_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    streaming_worker.log_failure_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+
+
+def test_redis_counts_keep_a_refreshed_ttl() -> None:
+    shared: Final = SharedRedisCounters()
+    worker: Final = _worker(shared)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert shared.count(f"{GROUP}_request_count:dep-a") == 0
+    assert shared.ttls == {f"{GROUP}_request_count:dep-a": IN_FLIGHT_COUNT_TTL_SECONDS}
+
+
+@pytest.mark.asyncio
+async def test_counts_stay_in_memory_without_redis() -> None:
+    worker: Final = _worker(None)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    await worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+    assert worker.router_cache.get_cache(f"{GROUP}_request_count:dep-a") == 0
+
+
+class UnavailableRedis(SharedRedisCounters):
+    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+        raise ConnectionError("redis is down")
+
+    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+        raise ConnectionError("redis is down")
+
+
+def test_redis_outage_never_fails_the_request() -> None:
+    worker: Final = _worker(UnavailableRedis())
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+
+
+def test_calls_without_a_deployment_are_ignored() -> None:
+    shared: Final = SharedRedisCounters()
+    worker: Final = _worker(shared)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs={"litellm_params": {"metadata": None}})
+    worker.log_pre_api_call(model="m", messages=[], kwargs={})
+
+    assert shared.encoded == {}
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index e7186dfe186..e413e6db2c6 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -1,9 +1,9 @@
 {
   "LIT001": {
-    "limit": 22180
+    "limit": 22176
   },
   "LIT002": {
-    "limit": 26729
+    "limit": 26721
   },
   "LIT003": {
     "limit": 261
@@ -27,10 +27,10 @@
     "limit": 0
   },
   "LIT010": {
-    "limit": 16426
+    "limit": 16412
   },
   "LIT011": {
-    "limit": 5506
+    "limit": 5505
   },
   "LIT012": {
     "limit": 4486

From c5aa4f07185c91f7c055e614734d5e216c6d043d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 21:14:58 -0700
Subject: [PATCH 05/27] fix(least-busy): fall back to per-worker counts when
 Redis is unreadable

Keep each worker's own in-flight counter up to date alongside the shared one,
so a Redis outage routes on that worker's counts the way it did before this
branch instead of treating every deployment as idle.

Floor a counter at zero when a decrement finds the key gone, which happens when
a request outlives the 1 hour TTL, so an expired counter cannot settle at -1 and
win every pick.
---
 litellm/router_strategy/least_busy.py         | 97 +++++++++++--------
 .../router_strategy/test_least_busy.py        | 39 +++++++-
 2 files changed, 95 insertions(+), 41 deletions(-)

diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index de6a4c4f59a..00d27b5f8d9 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -39,7 +39,7 @@ class _Deployment(TypedDict):
 _CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
 _DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
 _REDIS_COUNTS: Final = TypeAdapter(dict[str, float | None])
-_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...])
+_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None)
 
 
 def _request_count_key(model_group: str, deployment_id: str) -> str:
@@ -68,8 +68,20 @@ def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[
     )
 
 
-def _as_count(value: float | None) -> int:
-    return 0 if value is None else int(value)
+def _as_counts(values: Sequence[float | None]) -> tuple[int, ...]:
+    return tuple(0 if value is None else int(value) for value in values)
+
+
+def _shared_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
+    by_key: Final = _REDIS_COUNTS.validate_python(raw)
+    return _as_counts([by_key.get(key) for key in keys])
+
+
+def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
+    values: Final = _MEMORY_COUNTS.validate_python(raw)
+    if values is None or len(values) != len(keys):
+        return (0,) * len(keys)
+    return _as_counts(values)
 
 
 def _least_busy(
@@ -82,7 +94,8 @@ def _least_busy(
 
 def _warn_unreadable(model_group: str, error: Exception) -> None:
     verbose_router_logger.warning(
-        "least-busy routing could not read the in-flight counts for %s, treating every deployment as idle: %s",
+        "least-busy routing could not read the shared in-flight counts for %s, "
+        "falling back to this worker's own counts: %s",
         model_group,
         error,
     )
@@ -135,23 +148,31 @@ class LeastBusyLoggingHandler(CustomLogger):
         self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
     ) -> Mapping[str, object] | None:
         keys: Final = _request_count_keys(model_group, healthy_deployments)
-        try:
-            counts: Final = tuple(_as_count(value) for value in self._read_counts(keys))
-        except Exception as e:
-            _warn_unreadable(model_group, e)
-            return _least_busy(healthy_deployments, (0,) * len(keys))
-        return _least_busy(healthy_deployments, counts)
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is not None:
+            try:
+                shared: Final = _shared_counts(redis_cache.batch_get_cache(key_list=list(keys)), keys)
+            except Exception as e:
+                _warn_unreadable(model_group, e)
+            else:
+                return _least_busy(healthy_deployments, shared)
+        local: Final = _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)
+        return _least_busy(healthy_deployments, local)
 
     async def async_get_available_deployments(
         self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
     ) -> Mapping[str, object] | None:
         keys: Final = _request_count_keys(model_group, healthy_deployments)
-        try:
-            counts: Final = tuple(_as_count(value) for value in await self._async_read_counts(keys))
-        except Exception as e:
-            _warn_unreadable(model_group, e)
-            return _least_busy(healthy_deployments, (0,) * len(keys))
-        return _least_busy(healthy_deployments, counts)
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is not None:
+            try:
+                shared: Final = _shared_counts(await redis_cache.async_batch_get_cache(key_list=list(keys)), keys)
+            except Exception as e:
+                _warn_unreadable(model_group, e)
+            else:
+                return _least_busy(healthy_deployments, shared)
+        local: Final = _local_counts(await self.router_cache.async_batch_get_cache(list(keys), local_only=True), keys)
+        return _least_busy(healthy_deployments, local)
 
     def _increment(self, kwargs: Mapping[str, object], delta: int) -> None:
         ref: Final = _deployment_ref(kwargs)
@@ -160,10 +181,16 @@ class LeastBusyLoggingHandler(CustomLogger):
         key: Final = _request_count_key(*ref)
         redis_cache: Final = self.router_cache.redis_cache
         try:
+            local: Final = self.router_cache.increment_cache(
+                key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
+            )
+            if local < 0:
+                self.router_cache.set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
-                self.router_cache.increment_cache(key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
-            else:
-                redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+                return
+            shared: Final = redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+            if shared < 0:
+                redis_cache.set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
 
@@ -174,27 +201,17 @@ class LeastBusyLoggingHandler(CustomLogger):
         key: Final = _request_count_key(*ref)
         redis_cache: Final = self.router_cache.redis_cache
         try:
+            local: Final = await self.router_cache.async_increment_cache(
+                key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
+            )
+            if local is not None and local < 0:
+                await self.router_cache.async_set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
-                await self.router_cache.async_increment_cache(
-                    key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
-                )
-            else:
-                await redis_cache.async_increment(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+                return
+            shared: Final = await redis_cache.async_increment(
+                key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True
+            )
+            if shared < 0:
+                await redis_cache.async_set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
-
-    def _read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
-        redis_cache: Final = self.router_cache.redis_cache
-        if redis_cache is None:
-            return _MEMORY_COUNTS.validate_python(self.router_cache.batch_get_cache(list(keys), local_only=True))
-        by_key: Final = _REDIS_COUNTS.validate_python(redis_cache.batch_get_cache(key_list=list(keys)))
-        return tuple(by_key.get(key) for key in keys)
-
-    async def _async_read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
-        redis_cache: Final = self.router_cache.redis_cache
-        if redis_cache is None:
-            return _MEMORY_COUNTS.validate_python(
-                await self.router_cache.async_batch_get_cache(list(keys), local_only=True)
-            )
-        by_key: Final = _REDIS_COUNTS.validate_python(await redis_cache.async_batch_get_cache(key_list=list(keys)))
-        return tuple(by_key.get(key) for key in keys)
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
index 32a226a5d5b..55702c6fe73 100644
--- a/tests/test_litellm/router_strategy/test_least_busy.py
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -133,14 +133,51 @@ class UnavailableRedis(SharedRedisCounters):
         raise ConnectionError("redis is down")
 
 
-def test_redis_outage_never_fails_the_request() -> None:
+@pytest.mark.asyncio
+async def test_a_redis_outage_falls_back_to_this_workers_own_counts() -> None:
     worker: Final = _worker(UnavailableRedis())
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
 
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    await worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
     assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
 
 
+def test_a_shared_counter_that_expired_mid_request_cannot_go_negative() -> None:
+    shared: Final = SharedRedisCounters()
+    worker: Final = _worker(shared)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    shared.encoded.clear()
+    worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert shared.count(f"{GROUP}_request_count:dep-a") == 0
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+
+@pytest.mark.asyncio
+async def test_a_local_counter_that_expired_mid_request_cannot_go_negative() -> None:
+    worker: Final = _worker(None)
+    in_memory: Final = worker.router_cache.in_memory_cache
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    in_memory.delete_cache(f"{GROUP}_request_count:dep-a")
+    await worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert worker.router_cache.get_cache(f"{GROUP}_request_count:dep-a") == 0
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+
 def test_calls_without_a_deployment_are_ignored() -> None:
     shared: Final = SharedRedisCounters()
     worker: Final = _worker(shared)

From a9bc2cb50b4ba443e72b7d03a33447cc8724b037 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 22:38:26 -0700
Subject: [PATCH 06/27] fix(least-busy): clamp the shared in-flight count to
 zero in one call

A decrement whose matching increment is gone, because the counter key
expired while the request was still in flight, used to recreate the key
at -1, and a deployment with a negative count looks permanently idle, so
it collects every pick from then on. The repair write that followed the
decrement could also land after another pod's increment and erase it.

The increment, the clamp at zero and the TTL refresh now run as a single
Lua call, so nothing can interleave between them.
---
 litellm/caching/redis_cache.py                | 39 ++++++++++++++++++-
 litellm/router_strategy/least_busy.py         | 10 +----
 .../router_strategy/test_least_busy.py        | 20 +++++-----
 3 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index d4c47914394..4eccde5742b 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -20,6 +20,8 @@ from contextvars import ContextVar
 from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast
 
+from pydantic import TypeAdapter
+
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.constants import (
@@ -80,11 +82,22 @@ class _AsyncRedisCommands(Protocol):
 
     def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
 
+    def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
+
 
 _BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
     {"<lambda>", "wrapper", "_run_under_circuit_breaker", "_run_under_circuit_breaker_sync"}
 )
 
+_INCREMENT_WITH_FLOOR_LUA: Final = (
+    "local count = redis.call('INCRBY', KEYS[1], ARGV[1]) "
+    "if count < 0 then redis.call('SET', KEYS[1], 0) count = 0 end "
+    "redis.call('EXPIRE', KEYS[1], ARGV[2]) "
+    "return count"
+)
+
+_LUA_COUNT: Final = TypeAdapter(int)
+
 
 def _get_call_stack_info(num_frames: int = 2) -> str:
     """
@@ -680,7 +693,7 @@ class RedisCache(BaseCache):
             # NON blocking - notify users Redis is throwing an exception
             print_verbose(f"litellm.caching.caching: set() - Got exception from REDIS : {e}")
 
-    def increment_cache(self, key, value: int, ttl: float | None = None, refresh_ttl: bool = False, **kwargs) -> int:
+    def increment_cache(self, key, value: int, ttl: float | None = None, **kwargs) -> int:
         _redis_client: Final = self.redis_client
         start_time = time.time()
         set_ttl: Final = self.get_ttl(ttl=ttl)
@@ -701,7 +714,7 @@ class RedisCache(BaseCache):
             if set_ttl is not None:
                 # check if key already has ttl, if not -> set ttl
                 start_time = time.time()
-                current_ttl: Final = -1 if refresh_ttl else _redis_client.ttl(key)
+                current_ttl: Final = _redis_client.ttl(key)
                 end_time = time.time()
                 _duration = end_time - start_time
                 self.service_logger_obj.service_success_hook(
@@ -736,6 +749,20 @@ class RedisCache(BaseCache):
             )
             raise e
 
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+        """Add ``value`` to ``key``, clamp the result at zero, and refresh the TTL, in one Lua call.
+
+        A counter whose key expired while a request was still in flight would otherwise be
+        recreated negative by that request's decrement. Clamping inside the same call is what
+        keeps it safe: a separate corrective write could land after another pod's increment and
+        erase it. Returns the resulting count.
+        """
+        namespaced_key: Final = self.check_and_fix_namespace(key=key)
+        count: Final[object] = self.redis_client.eval(  # pyright: ignore[reportAttributeAccessIssue]  # stubs omit eval
+            _INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl
+        )
+        return _LUA_COUNT.validate_python(count)
+
     @_redis_circuit_breaker_guard
     async def async_scan_iter(self, pattern: str, count: int = 100) -> list:
         start_time: Final = time.time()
@@ -1241,6 +1268,14 @@ class RedisCache(BaseCache):
             result = result.decode()
         return float(result)
 
+    @_redis_circuit_breaker_guard
+    async def async_increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+        """Async twin of ``increment_with_floor``, sharing its Lua script and its guarantees."""
+        _redis_client: Final = self._async_commands()
+        namespaced_key: Final = self.check_and_fix_namespace(key=key)
+        count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
+        return _LUA_COUNT.validate_python(count)
+
     async def flush_cache_buffer(self):
         print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
         await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index 00d27b5f8d9..771d2bb4328 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -188,9 +188,7 @@ class LeastBusyLoggingHandler(CustomLogger):
                 self.router_cache.set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
                 return
-            shared: Final = redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
-            if shared < 0:
-                redis_cache.set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
+            redis_cache.increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
 
@@ -208,10 +206,6 @@ class LeastBusyLoggingHandler(CustomLogger):
                 await self.router_cache.async_set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
                 return
-            shared: Final = await redis_cache.async_increment(
-                key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True
-            )
-            if shared < 0:
-                await redis_cache.async_set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
+            await redis_cache.async_increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
index 55702c6fe73..eb9591811ac 100644
--- a/tests/test_litellm/router_strategy/test_least_busy.py
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -40,17 +40,16 @@ class SharedRedisCounters:
     async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
         self.set_cache(key, value)
 
-    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         current: Final = self.count(key) or 0
         assert isinstance(current, int)
-        incremented: Final = current + value
+        incremented: Final = max(0, current + value)
         self.encoded[key] = json.dumps(incremented)
-        if ttl is not None and (refresh_ttl or key not in self.ttls):
-            self.ttls[key] = ttl
+        self.ttls[key] = ttl
         return incremented
 
-    async def async_increment(self, key: str, value: float, ttl: int | None = None, refresh_ttl: bool = False) -> float:
-        return self.increment_cache(key, int(value), ttl, refresh_ttl)
+    async def async_increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+        return self.increment_with_floor(key, value, ttl)
 
     def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
         return {key: self.count(key) for key in key_list}
@@ -102,12 +101,14 @@ def test_sync_pick_reads_the_shared_counts() -> None:
 def test_redis_counts_keep_a_refreshed_ttl() -> None:
     shared: Final = SharedRedisCounters()
     worker: Final = _worker(shared)
+    key: Final = f"{GROUP}_request_count:dep-a"
+    shared.ttls[key] = 5
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
     worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
 
-    assert shared.count(f"{GROUP}_request_count:dep-a") == 0
-    assert shared.ttls == {f"{GROUP}_request_count:dep-a": IN_FLIGHT_COUNT_TTL_SECONDS}
+    assert shared.count(key) == 0
+    assert shared.ttls == {key: IN_FLIGHT_COUNT_TTL_SECONDS}
 
 
 @pytest.mark.asyncio
@@ -129,7 +130,7 @@ class UnavailableRedis(SharedRedisCounters):
     def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
         raise ConnectionError("redis is down")
 
-    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         raise ConnectionError("redis is down")
 
 
@@ -159,6 +160,7 @@ def test_a_shared_counter_that_expired_mid_request_cannot_go_negative() -> None:
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
 
+    assert shared.count(f"{GROUP}_request_count:dep-a") == 1
     assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
 
 

From caf9bbbd5a535e61da844e1a0225a4f8a6312b8d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 6 Sep 2026 00:20:05 -0700
Subject: [PATCH 07/27] fix(least-busy): keep the shared count readable,
 counted once, and off the loop

A Redis outage read as "every deployment is idle", because batch_get_cache
swallows the failure and answers with an empty dict. batch_get_counts and its
async twin raise instead, so a worker that cannot reach Redis falls back to its
own numbers rather than routing on zeros.

The counter's TTL is now set only on a key that has none, so a +1 left behind by
a worker that died mid-request ages out an hour after the key was created. It
used to be refreshed on every touch, which kept that stuck count alive for as
long as the group took traffic.

Two least-busy groups counted the same request twice, since the pre-call list
kept a selector per group while the success list deduped by class. The selector
now goes on through add_litellm_input_callback, which dedupes the same way.

A prompt-management model picked its deployment on the synchronous path, so the
new Redis read landed on the event loop and configured routing plugins never
ran. It awaits the async selector now.
---
 basedpyright-code-budget.json                 | 12 ++--
 litellm/caching/redis_cache.py                | 36 +++++++++-
 litellm/router.py                             |  8 ++-
 litellm/router_strategy/least_busy.py         | 10 +--
 ruff-strict-budget.json                       | 10 +--
 .../test_litellm/caching/test_redis_cache.py  | 44 ++++++++++++
 .../router_strategy/test_least_busy.py        | 67 +++++++++----------
 .../test_router_routing_groups.py             | 38 +++++++++++
 .../test_router_routing_plugins.py            | 30 +++++++++
 type-discipline-budget.json                   |  8 +--
 10 files changed, 199 insertions(+), 64 deletions(-)

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index c21dc76743a..32d3730aa32 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5551
+    "limit": 5532
   },
   "reportMissingTypeArgument": {
-    "limit": 15277
+    "limit": 15273
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -105,13 +105,13 @@
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38240
+    "limit": 38208
   },
   "reportUnknownParameterType": {
-    "limit": 19558
+    "limit": 19532
   },
   "reportUnknownVariableType": {
-    "limit": 29781
+    "limit": 29751
   },
   "reportUnnecessaryCast": {
     "limit": 110
@@ -135,7 +135,7 @@
     "limit": 21
   },
   "reportUnusedFunction": {
-    "limit": 138
+    "limit": 137
   },
   "reportUnusedImport": {
     "limit": 542
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index 4eccde5742b..3abc6e6f3c9 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -92,11 +92,18 @@ _BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
 _INCREMENT_WITH_FLOOR_LUA: Final = (
     "local count = redis.call('INCRBY', KEYS[1], ARGV[1]) "
     "if count < 0 then redis.call('SET', KEYS[1], 0) count = 0 end "
-    "redis.call('EXPIRE', KEYS[1], ARGV[2]) "
+    "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]) end "
     "return count"
 )
 
 _LUA_COUNT: Final = TypeAdapter(int)
+_OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
+
+
+def _decoded_counts(values: Sequence[bytes | str | None]) -> tuple[int | None, ...]:
+    return _OPTIONAL_COUNTS.validate_python(
+        tuple(value.decode("utf-8") if isinstance(value, bytes) else value for value in values)
+    )
 
 
 def _get_call_stack_info(num_frames: int = 2) -> str:
@@ -749,13 +756,19 @@ class RedisCache(BaseCache):
             )
             raise e
 
+    @_redis_circuit_breaker_guard_sync
     def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
-        """Add ``value`` to ``key``, clamp the result at zero, and refresh the TTL, in one Lua call.
+        """Add ``value`` to ``key``, clamp the result at zero, and give a new key ``ttl``, in one Lua call.
 
         A counter whose key expired while a request was still in flight would otherwise be
         recreated negative by that request's decrement. Clamping inside the same call is what
         keeps it safe: a separate corrective write could land after another pod's increment and
-        erase it. Returns the resulting count.
+        erase it.
+
+        The TTL is set only on a key that has none, so a counter expires ``ttl`` after it was
+        created rather than ``ttl`` after it was last touched. Refreshing it on every touch
+        would keep a count a dead worker never decremented alive for as long as the group
+        takes traffic. Returns the resulting count.
         """
         namespaced_key: Final = self.check_and_fix_namespace(key=key)
         count: Final[object] = self.redis_client.eval(  # pyright: ignore[reportAttributeAccessIssue]  # stubs omit eval
@@ -763,6 +776,23 @@ class RedisCache(BaseCache):
         )
         return _LUA_COUNT.validate_python(count)
 
+    @_redis_circuit_breaker_guard_sync
+    def batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        """Read integer counters for ``key_list``, in order, raising when Redis cannot answer.
+
+        ``batch_get_cache`` swallows every failure and returns an empty dict, which the caller
+        cannot tell apart from "every counter is unset". A caller that has to fall back to its
+        own numbers when Redis is unreachable needs the failure, not a dict of zeros.
+        """
+        namespaced_keys: Final = [self.check_and_fix_namespace(key=key) for key in key_list]
+        return _decoded_counts(self._run_redis_mget_operation(keys=namespaced_keys))
+
+    @_redis_circuit_breaker_guard
+    async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        """Async twin of ``batch_get_counts``, raising on failure the same way."""
+        namespaced_keys: Final = [self.check_and_fix_namespace(key=key) for key in key_list]
+        return _decoded_counts(await self._async_run_redis_mget_operation(keys=namespaced_keys))
+
     @_redis_circuit_breaker_guard
     async def async_scan_iter(self, pattern: str, count: int = 100) -> list:
         start_time: Final = time.time()
diff --git a/litellm/router.py b/litellm/router.py
index 3f450661946..f5b7924fb58 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -1212,7 +1212,7 @@ class Router:
                 selector = LeastBusyLoggingHandler(router_cache=self.cache)
                 if register_callbacks:
                     if isinstance(litellm.input_callback, list):
-                        litellm.input_callback.append(selector)
+                        litellm.logging_callback_manager.add_litellm_input_callback(selector)
                     else:
                         litellm.input_callback = [selector]
             case RoutingStrategy.USAGE_BASED_ROUTING.value:
@@ -4139,10 +4139,12 @@ class Router:
                 }
             )
         litellm_logging_object = cast(LiteLLMLogging, litellm_logging_object)
-        prompt_management_deployment: Final = self.get_available_deployment(
+        specific_deployment: Final = kwargs.pop("specific_deployment", None)
+        prompt_management_deployment: Final = await self.async_get_available_deployment(
             model=model,
             messages=[{"role": "user", "content": "prompt"}],
-            specific_deployment=kwargs.pop("specific_deployment", None),
+            specific_deployment=specific_deployment,
+            request_kwargs=kwargs,
         )
 
         self._update_kwargs_with_deployment(deployment=prompt_management_deployment, kwargs=kwargs)
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index 771d2bb4328..ab6d702bbc2 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -38,7 +38,6 @@ class _Deployment(TypedDict):
 
 _CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
 _DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
-_REDIS_COUNTS: Final = TypeAdapter(dict[str, float | None])
 _MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None)
 
 
@@ -72,11 +71,6 @@ def _as_counts(values: Sequence[float | None]) -> tuple[int, ...]:
     return tuple(0 if value is None else int(value) for value in values)
 
 
-def _shared_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
-    by_key: Final = _REDIS_COUNTS.validate_python(raw)
-    return _as_counts([by_key.get(key) for key in keys])
-
-
 def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
     values: Final = _MEMORY_COUNTS.validate_python(raw)
     if values is None or len(values) != len(keys):
@@ -151,7 +145,7 @@ class LeastBusyLoggingHandler(CustomLogger):
         redis_cache: Final = self.router_cache.redis_cache
         if redis_cache is not None:
             try:
-                shared: Final = _shared_counts(redis_cache.batch_get_cache(key_list=list(keys)), keys)
+                shared: Final = _as_counts(redis_cache.batch_get_counts(list(keys)))
             except Exception as e:
                 _warn_unreadable(model_group, e)
             else:
@@ -166,7 +160,7 @@ class LeastBusyLoggingHandler(CustomLogger):
         redis_cache: Final = self.router_cache.redis_cache
         if redis_cache is not None:
             try:
-                shared: Final = _shared_counts(await redis_cache.async_batch_get_cache(key_list=list(keys)), keys)
+                shared: Final = _as_counts(await redis_cache.async_batch_get_counts(list(keys)))
             except Exception as e:
                 _warn_unreadable(model_group, e)
             else:
diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index ceff8c6e15c..d63a69de76f 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -1,6 +1,6 @@
 {
   "ANN001": {
-    "limit": 2937
+    "limit": 2918
   },
   "ANN002": {
     "limit": 71
@@ -9,10 +9,10 @@
     "limit": 806
   },
   "ANN201": {
-    "limit": 1972
+    "limit": 1965
   },
   "ANN202": {
-    "limit": 830
+    "limit": 829
   },
   "ANN204": {
     "limit": 683
@@ -57,7 +57,7 @@
     "limit": 3
   },
   "BLE001": {
-    "limit": 2915
+    "limit": 2914
   },
   "C401": {
     "limit": 8
@@ -189,7 +189,7 @@
     "limit": 0
   },
   "S110": {
-    "limit": 212
+    "limit": 207
   },
   "S112": {
     "limit": 22
diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py
index 2f412e7382b..6b2df118611 100644
--- a/tests/test_litellm/caching/test_redis_cache.py
+++ b/tests/test_litellm/caching/test_redis_cache.py
@@ -525,6 +525,50 @@ def test_circuit_breaker_open_keeps_sync_batch_get_cache_as_a_miss(sync_batch_re
     assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {}
 
 
+def test_batch_get_counts_raises_where_batch_get_cache_reports_a_miss(sync_batch_redis_cache):
+    """A caller that must fall back when Redis is unreachable needs the failure, not zeros.
+
+    The batch read answers a dead Redis with an empty dict, which a counting caller cannot tell
+    apart from "every counter is unset". Least-busy routing read that as an idle deployment and
+    kept sending traffic to it instead of falling back to this worker's own in-flight counts.
+    """
+    assert sync_batch_redis_cache.batch_get_cache(key_list=["lit7039"]) == {}
+
+    with pytest.raises(OSError, match="redis unavailable"):
+        sync_batch_redis_cache.batch_get_counts(["lit7039"])
+
+
+@pytest.mark.asyncio
+async def test_async_batch_get_counts_raises_where_async_batch_get_cache_reports_a_miss(redis_no_ping: None):
+    """Async twin: the async batch read hides the same failure behind an empty dict."""
+    failing_client = AsyncMock()
+    failing_client.mget.side_effect = OSError("redis unavailable")
+    with patch(  # test-quality-ok: RedisCache.__init__ builds its client eagerly, with no injection point
+        "litellm._redis.get_redis_client", return_value=MagicMock()
+    ):
+        cache = RedisCache(host="127.0.0.1", port=6379)
+
+    with patch.object(cache, "init_async_client", return_value=failing_client):
+        assert await cache.async_batch_get_cache(key_list=["lit7039"]) == {}
+
+        with pytest.raises(OSError, match="redis unavailable"):
+            await cache.async_batch_get_counts(["lit7039"])
+
+
+@pytest.mark.parametrize("stored", [b"3", "3"])
+def test_batch_get_counts_reads_counters_in_order_and_keeps_unset_keys_apart(stored, redis_no_ping: None):
+    """Counters come back positionally, so an unset key has to stay a hole rather than shift the
+    rest of the row onto the wrong deployments, and a count has to survive whether the client
+    hands it back as bytes or as text."""
+    with patch(  # test-quality-ok: RedisCache.__init__ builds its client eagerly, with no injection point
+        "litellm._redis.get_redis_client", return_value=MagicMock()
+    ):
+        cache = RedisCache(host="127.0.0.1", port=6379)
+    cache.redis_client.mget.return_value = [stored, None, b"0"]
+
+    assert cache.batch_get_counts(["dep-a", "dep-b", "dep-c"]) == (3, None, 0)
+
+
 @pytest.fixture
 def sync_batch_cache_with_service_logger(redis_no_ping: None) -> Iterator[tuple[RedisCache, ServiceLogging]]:
     service_logger = ServiceLogging(mock_testing=True)
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
index eb9591811ac..9efa526fc02 100644
--- a/tests/test_litellm/router_strategy/test_least_busy.py
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -1,4 +1,3 @@
-import json
 from typing import Final
 
 import pytest
@@ -18,44 +17,34 @@ def _call_kwargs(deployment_id: str) -> dict[str, object]:
 
 
 class SharedRedisCounters:
-    """Stores JSON strings and hands back a fresh object per read, the way a real Redis client does."""
+    """Mirrors what Redis gives the handler: increments clamped at zero, a TTL set once when
+    the key is created, and ordered reads that raise rather than invent a value."""
 
     def __init__(self) -> None:
-        self.encoded: dict[str, str] = {}
-        self.ttls: dict[str, float] = {}
+        self.counts: dict[str, int] = {}
+        self.ttls: dict[str, int] = {}
 
-    def count(self, key: str) -> object:
-        raw: Final = self.encoded.get(key)
-        return None if raw is None else json.loads(raw)
+    def count(self, key: str) -> int | None:
+        return self.counts.get(key)
 
-    def get_cache(self, key: str, **kwargs: object) -> object:
-        return self.count(key)
-
-    def set_cache(self, key: str, value: object, **kwargs: object) -> None:
-        self.encoded[key] = json.dumps(value)
-
-    async def async_get_cache(self, key: str, **kwargs: object) -> object:
-        return self.count(key)
-
-    async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
-        self.set_cache(key, value)
+    def expire(self, key: str) -> None:
+        self.counts.pop(key, None)
+        self.ttls.pop(key, None)
 
     def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
-        current: Final = self.count(key) or 0
-        assert isinstance(current, int)
-        incremented: Final = max(0, current + value)
-        self.encoded[key] = json.dumps(incremented)
-        self.ttls[key] = ttl
+        incremented: Final = max(0, self.counts.get(key, 0) + value)
+        self.counts[key] = incremented
+        self.ttls.setdefault(key, ttl)
         return incremented
 
     async def async_increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         return self.increment_with_floor(key, value, ttl)
 
-    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
-        return {key: self.count(key) for key in key_list}
+    def batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        return tuple(self.counts.get(key) for key in key_list)
 
-    async def async_batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
-        return self.batch_get_cache(key_list)
+    async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        return self.batch_get_counts(key_list)
 
 
 def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler:
@@ -98,17 +87,25 @@ def test_sync_pick_reads_the_shared_counts() -> None:
     assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
 
 
-def test_redis_counts_keep_a_refreshed_ttl() -> None:
+def test_the_handler_never_pushes_a_counters_ttl_forward() -> None:
+    """A worker that dies mid-request leaves a +1 nobody will ever decrement. Redis expires that
+    stuck count an hour after the key was created, which only works while nothing writes the TTL
+    again: a handler that refreshed it on every touch would keep the count alive for as long as
+    the group takes traffic, and the deployment would read busier than it is forever."""
     shared: Final = SharedRedisCounters()
     worker: Final = _worker(shared)
     key: Final = f"{GROUP}_request_count:dep-a"
-    shared.ttls[key] = 5
 
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert shared.ttls == {key: IN_FLIGHT_COUNT_TTL_SECONDS}
+
+    shared.ttls[key] = 5
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
     worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
 
-    assert shared.count(key) == 0
-    assert shared.ttls == {key: IN_FLIGHT_COUNT_TTL_SECONDS}
+    assert shared.count(key) == 1
+    assert shared.ttls == {key: 5}
 
 
 @pytest.mark.asyncio
@@ -127,10 +124,10 @@ async def test_counts_stay_in_memory_without_redis() -> None:
 
 
 class UnavailableRedis(SharedRedisCounters):
-    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         raise ConnectionError("redis is down")
 
-    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+    def batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
         raise ConnectionError("redis is down")
 
 
@@ -153,7 +150,7 @@ def test_a_shared_counter_that_expired_mid_request_cannot_go_negative() -> None:
     worker: Final = _worker(shared)
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
-    shared.encoded.clear()
+    shared.expire(f"{GROUP}_request_count:dep-a")
     worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
 
     assert shared.count(f"{GROUP}_request_count:dep-a") == 0
@@ -187,4 +184,4 @@ def test_calls_without_a_deployment_are_ignored() -> None:
     worker.log_pre_api_call(model="m", messages=[], kwargs={"litellm_params": {"metadata": None}})
     worker.log_pre_api_call(model="m", messages=[], kwargs={})
 
-    assert shared.encoded == {}
+    assert shared.counts == {}
diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py
index 5599c5aad63..0ce0ed5b37e 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_groups.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py
@@ -12,6 +12,7 @@ import pytest
 
 import litellm
 from litellm import Router
+from litellm.integrations.custom_logger import CustomLogger
 from litellm.types.router import RoutingGroup, RoutingStrategy
 
 
@@ -435,6 +436,43 @@ def test_update_settings_unregisters_group_selectors_when_groups_removed(monkeyp
     assert router._group_selectors == {}
 
 
+def test_two_least_busy_groups_count_a_request_once(monkeypatch):
+    """
+    Least-busy counts a request up from the pre-call hooks on `litellm.input_callback` and
+    back down from the success hooks on `litellm.callbacks`. The success list drops a second
+    selector of the same class, so a pre-call list that kept both counted every request twice
+    and released it once, and the deployment's in-flight count climbed until it looked pinned.
+    """
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm, "input_callback", [])
+
+    router = _build_router(
+        routing_strategy="least-busy",
+        routing_groups=[
+            {
+                "group_name": "fast",
+                "models": ["filtered-model"],
+                "routing_strategy": "least-busy",
+            }
+        ],
+    )
+    kwargs = {
+        "litellm_params": {
+            "metadata": {"model_group": "filtered-model"},
+            "model_info": {"id": "deploy-1"},
+        }
+    }
+
+    for callback in litellm.input_callback:
+        if isinstance(callback, CustomLogger):
+            callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs)
+    for callback in litellm.callbacks:
+        if isinstance(callback, CustomLogger):
+            callback.log_success_event(kwargs, None, None, None)
+
+    assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0
+
+
 # ---------------------------------------------------------------------------
 # Direct helper coverage
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/router_strategy/test_router_routing_plugins.py b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
index 293af36080a..c49b22ea367 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_plugins.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
@@ -164,6 +164,36 @@ async def test_async_completion_with_unsupported_strategy_rejects_configured_plu
         await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}])
 
 
+@pytest.mark.asyncio
+async def test_prompt_management_model_still_runs_the_plugin_pipeline():
+    """
+    A prompt-management model routes through its own factory, which picked the deployment
+    on the synchronous path. Plugins never run there, so the guard turned every such request
+    into an error message about the caller's own API choice, on an async call the caller made
+    correctly. It also read the in-flight counts with a blocking call inside the event loop.
+    """
+    router = Router(
+        model_list=[
+            {
+                "model_name": "cached-claude",
+                "litellm_params": {
+                    "model": "anthropic_cache_control_hook/claude-sonnet-5",
+                    "prompt_id": "cache-points",
+                },
+            }
+        ],
+        routing_strategy="least-busy",
+        plugins=[BlockEverything()],
+    )
+
+    with pytest.raises(ValueError, match="No deployments left after routing-plugin filtering"):
+        await router.acompletion(
+            model="cached-claude",
+            messages=[{"role": "user", "content": "hi"}],
+            litellm_call_id="lit-7039",
+        )
+
+
 @pytest.mark.asyncio
 async def test_router_without_plugins_is_unaffected():
     """Regression guard: a Router with no `plugins` configured behaves exactly as before."""
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index e413e6db2c6..0c0952289e2 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -1,9 +1,9 @@
 {
   "LIT001": {
-    "limit": 22176
+    "limit": 22174
   },
   "LIT002": {
-    "limit": 26721
+    "limit": 26715
   },
   "LIT003": {
     "limit": 261
@@ -27,10 +27,10 @@
     "limit": 0
   },
   "LIT010": {
-    "limit": 16412
+    "limit": 16398
   },
   "LIT011": {
-    "limit": 5505
+    "limit": 5504
   },
   "LIT012": {
     "limit": 4486

From ef5f51abca77d3081d04796ce9b3191dc2409f9b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 6 Sep 2026 01:59:15 -0700
Subject: [PATCH 08/27] fix(least-busy): count for every router, and keep the
 expiry through a clamp

Two routers in one process shared a single handler, because the callback
manager dedupes on the class name plus the handler's public attributes and the
handler had none. The second router's requests were never counted. The handler
now carries the id of the cache it was built on, so routers with different
caches both register while the two selectors one router builds for its routing
groups still collapse into one.

Clamping a negative count back to zero used SET, which drops the key's TTL, so
the next write started the hour over. It uses INCRBY by the negative amount now,
which leaves the expiry alone.

The Lua script had no test that ran it, so tests/local_testing covers both the
sync and async paths against a real Redis, and the file is wired into the
CircleCI job that provides one.
---
 .circleci/config.yml                          |  1 +
 litellm/caching/redis_cache.py                |  2 +-
 litellm/router_strategy/least_busy.py         |  1 +
 .../test_redis_increment_with_floor.py        | 80 +++++++++++++++++++
 .../test_router_routing_groups.py             | 38 +++++++++
 5 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 tests/local_testing/test_redis_increment_with_floor.py

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6e368a3debe..d10962864bf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1440,6 +1440,7 @@ jobs:
             TEST_FILES=$(printf "%s\n" \
               tests/local_testing/test_dual_cache.py \
               tests/local_testing/test_redis_batch_optimizations.py \
+              tests/local_testing/test_redis_increment_with_floor.py \
               tests/local_testing/test_router_utils.py)
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index 3abc6e6f3c9..106c1580110 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -91,7 +91,7 @@ _BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
 
 _INCREMENT_WITH_FLOOR_LUA: Final = (
     "local count = redis.call('INCRBY', KEYS[1], ARGV[1]) "
-    "if count < 0 then redis.call('SET', KEYS[1], 0) count = 0 end "
+    "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count) end "
     "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]) end "
     "return count"
 )
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index ab6d702bbc2..14e6592e1fd 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -106,6 +106,7 @@ class LeastBusyLoggingHandler(CustomLogger):
 
     def __init__(self, router_cache: DualCache):
         self.router_cache = router_cache
+        self.router_cache_id = str(id(router_cache))
 
     def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
         self._increment(kwargs, 1)
diff --git a/tests/local_testing/test_redis_increment_with_floor.py b/tests/local_testing/test_redis_increment_with_floor.py
new file mode 100644
index 00000000000..e358d5f31e0
--- /dev/null
+++ b/tests/local_testing/test_redis_increment_with_floor.py
@@ -0,0 +1,80 @@
+"""Least-busy routing keeps its in-flight counters in Redis, and the clamp at zero plus the
+create-once TTL both live inside a Lua script. Nothing but a real Redis runs that script, so
+these are the only tests that fail when the script itself is wrong."""
+
+import os
+import uuid
+from typing import Final
+
+import pytest
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from litellm.caching.redis_cache import RedisCache
+
+TTL: Final = 600
+
+
+@pytest.fixture
+def counter():
+    cache: Final = RedisCache(host=os.getenv("REDIS_HOST"), port=os.getenv("REDIS_PORT"))
+    key: Final = f"lit7039-{uuid.uuid4()}"
+    yield cache, key, cache.check_and_fix_namespace(key=key)
+    cache.delete_cache(key)
+
+
+def test_a_counter_adds_every_increment_and_reads_back_what_it_holds(counter):
+    cache, key, _ = counter
+
+    assert cache.increment_with_floor(key, 3, TTL) == 3
+    assert cache.increment_with_floor(key, 2, TTL) == 5
+    assert cache.batch_get_counts([key]) == (5,)
+
+
+def test_a_decrement_past_zero_leaves_the_counter_at_zero(counter):
+    """A worker whose counter expired mid-request decrements a key that is no longer there.
+    Without the clamp that deployment reads negative, and least-busy pins every later request
+    on it until the count climbs back to zero."""
+    cache, key, _ = counter
+
+    assert cache.increment_with_floor(key, 1, TTL) == 1
+    assert cache.increment_with_floor(key, -5, TTL) == 0
+    assert cache.batch_get_counts([key]) == (0,)
+
+
+def test_traffic_never_pushes_a_counters_expiry_back_out(counter):
+    """The TTL is what releases a count whose worker died mid-request. Rewriting it on every
+    touch would keep that stuck count alive for as long as the group takes traffic."""
+    cache, key, namespaced_key = counter
+
+    cache.increment_with_floor(key, 1, TTL)
+    assert cache.redis_client.ttl(namespaced_key) > TTL - 60
+
+    cache.redis_client.expire(namespaced_key, 30)
+    cache.increment_with_floor(key, 1, TTL)
+
+    assert cache.redis_client.ttl(namespaced_key) <= 30
+
+
+def test_clamping_to_zero_keeps_the_expiry_it_already_had(counter):
+    cache, key, namespaced_key = counter
+
+    cache.increment_with_floor(key, 1, TTL)
+    cache.redis_client.expire(namespaced_key, 30)
+
+    assert cache.increment_with_floor(key, -5, TTL) == 0
+    assert cache.redis_client.ttl(namespaced_key) <= 30
+
+
+@pytest.mark.asyncio
+async def test_the_async_counter_behaves_the_same_way(counter):
+    cache, key, namespaced_key = counter
+
+    assert await cache.async_increment_with_floor(key, 2, TTL) == 2
+    assert await cache.async_batch_get_counts([key]) == (2,)
+
+    cache.redis_client.expire(namespaced_key, 30)
+
+    assert await cache.async_increment_with_floor(key, -9, TTL) == 0
+    assert cache.redis_client.ttl(namespaced_key) <= 30
diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py
index 0ce0ed5b37e..af390c3292b 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_groups.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py
@@ -473,6 +473,44 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch):
     assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0
 
 
+def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch):
+    """
+    Least-busy hangs its counting off litellm's global callback lists, and those lists keep one
+    logger per class unless the instances differ in a plain attribute. Two routers in one process
+    (a second Router, or a per-request `user_config` one) therefore have to register separately:
+    a second router whose selector is dropped counts nothing, reads zero for every deployment,
+    and sends every request to whichever one is listed first.
+    """
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm, "input_callback", [])
+
+    first = _build_router(routing_strategy="least-busy")
+    second = _build_router(routing_strategy="least-busy")
+    kwargs = {
+        "litellm_params": {
+            "metadata": {"model_group": "filtered-model"},
+            "model_info": {"id": "deploy-1"},
+        }
+    }
+
+    for callback in litellm.input_callback:
+        if isinstance(callback, CustomLogger):
+            callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs)
+
+    assert second.cache.get_cache("filtered-model_request_count:deploy-1") == 1
+    assert (
+        second.get_available_deployment(model="filtered-model", messages=[])["model_info"]["id"]
+        == "deploy-2"
+    )
+
+    for callback in litellm.callbacks:
+        if isinstance(callback, CustomLogger):
+            callback.log_success_event(kwargs, None, None, None)
+
+    assert first.cache.get_cache("filtered-model_request_count:deploy-1") == 0
+    assert second.cache.get_cache("filtered-model_request_count:deploy-1") == 0
+
+
 # ---------------------------------------------------------------------------
 # Direct helper coverage
 # ---------------------------------------------------------------------------

From 314e8905c583792d08207cdcd1bfa8e22d985f51 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 6 Sep 2026 02:44:45 -0700
Subject: [PATCH 09/27] fix(router): let prompt-management plugins see the
 caller's own messages

The prompt-management factory picks its deployment with a placeholder message. That
was inert while the pick ran on the synchronous path, which never runs the routing
plugin pipeline. Now that the pick runs the pipeline, a plugin classifying request
content would score the placeholder instead of the conversation, and the narrowing
it writes decides which deployments the real call may use.
---
 litellm/router.py                             |  2 +-
 .../test_router_routing_plugins.py            | 47 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/litellm/router.py b/litellm/router.py
index f5b7924fb58..398d87ce843 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -4142,7 +4142,7 @@ class Router:
         specific_deployment: Final = kwargs.pop("specific_deployment", None)
         prompt_management_deployment: Final = await self.async_get_available_deployment(
             model=model,
-            messages=[{"role": "user", "content": "prompt"}],
+            messages=cast(list[dict[str, str]], messages),  # cast-ok: selection reads messages structurally
             specific_deployment=specific_deployment,
             request_kwargs=kwargs,
         )
diff --git a/tests/test_litellm/router_strategy/test_router_routing_plugins.py b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
index c49b22ea367..0a54addce5e 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_plugins.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
@@ -56,6 +56,18 @@ class BlockEverything:
         return context
 
 
+class MessageRecorder:
+    """Records what each plugin pass was handed, then blocks so the request stops there."""
+
+    def __init__(self):
+        self.seen = []
+
+    async def run(self, context: RoutingContext) -> RoutingContext:
+        self.seen.append(list(context.raw_messages))
+        context.candidate_models = []
+        return context
+
+
 def _smart_router_model_list():
     return [
         {
@@ -194,6 +206,41 @@ async def test_prompt_management_model_still_runs_the_plugin_pipeline():
         )
 
 
+@pytest.mark.asyncio
+async def test_prompt_management_plugins_see_the_callers_own_messages():
+    """
+    The prompt-management factory picks its deployment with a placeholder message, which was
+    harmless while that pick ran on the synchronous path (plugins never ran there at all). Now
+    that the pick runs the plugin pipeline, a plugin that classifies request content would score
+    the placeholder instead of the conversation, and the narrowing it produces decides which
+    deployments the real call is allowed to use.
+    """
+    recorder = MessageRecorder()
+    router = Router(
+        model_list=[
+            {
+                "model_name": "cached-claude",
+                "litellm_params": {
+                    "model": "anthropic_cache_control_hook/claude-sonnet-5",
+                    "prompt_id": "cache-points",
+                },
+            }
+        ],
+        routing_strategy="least-busy",
+        plugins=[recorder],
+    )
+    messages = [{"role": "user", "content": "wire me $40,000 to account 12345"}]
+
+    with pytest.raises(ValueError, match="No deployments left after routing-plugin filtering"):
+        await router.acompletion(
+            model="cached-claude",
+            messages=messages,
+            litellm_call_id="lit-7039",
+        )
+
+    assert recorder.seen == [messages]
+
+
 @pytest.mark.asyncio
 async def test_router_without_plugins_is_unaffected():
     """Regression guard: a Router with no `plugins` configured behaves exactly as before."""

From 1a6aa98230571db22ccd37e5db2c6011a4f0c4c4 Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Mon, 7 Sep 2026 23:29:42 -0700
Subject: [PATCH 10/27] fix(spend): compare auto-router targets by deployment
 identity (#40206)

Preserve deployment identity through savings calculation, with canonical model fallback only when either ID is absent. Cover negotiated rates, unchanged deployments, alias/base-model cache accounting and missing IDs.

Fixes #38811. Based on the deployment-identity approach proposed by @QuantumBreakz in #38834.

Co-authored-by: Claude Code <noreply@anthropic.com>
---
 litellm/proxy/spend_tracking/savings.py       | 15 ++-
 .../proxy/spend_tracking/test_savings.py      | 98 +++++++++++++++++++
 2 files changed, 108 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py
index c541b9b40e5..950fcca2039 100644
--- a/litellm/proxy/spend_tracking/savings.py
+++ b/litellm/proxy/spend_tracking/savings.py
@@ -299,6 +299,8 @@ def compute_autorouter_savings(
     selected_info: ModelInfo | None = None,
     baseline_info: ModelInfo | None = None,
     cost_breakdown: Mapping[str, object] | None = None,
+    baseline_deployment_id: str | None = None,
+    selected_deployment_id: str | None = None,
 ) -> float:
     """Net dollars the router saved, or cost, by serving this request on ``selected_model``.
 
@@ -334,11 +336,12 @@ def compute_autorouter_savings(
     selected: Final = _resolve_model(selected_model, selected_provider)
     if baseline is None or selected is None:
         return 0.0
-    # Same model is only the same cost when it is also the same deployment. Two
-    # deployments of one model can carry different negotiated rates, and routing from
-    # the dear one to the cheap one is a real saving that short-circuiting on the model
-    # name alone reports as zero.
-    if baseline == selected:
+    same_target: Final = (
+        baseline_deployment_id == selected_deployment_id
+        if baseline_deployment_id and selected_deployment_id
+        else baseline == selected
+    )
+    if same_target:
         return 0.0
     basis: Final = _pricing_basis(cost_breakdown)
     effective_baseline_info: Final = baseline_info if baseline_info is not None else _model_info(baseline)
@@ -517,6 +520,8 @@ def autorouter_savings_for_request(
         selected_info=_effective_model_info(router_instance, model_id, model or ""),
         baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""),
         cost_breakdown=cost_breakdown,
+        baseline_deployment_id=baseline_id,
+        selected_deployment_id=model_id,
     )
     classifier_cost: Final = classifier_cost_from_decision(decision)
     return gross if classifier_cost is None else gross - classifier_cost
diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py
index cc8fdeb0160..e466edab131 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_savings.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py
@@ -1162,6 +1162,104 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
     assert result.prompt_caching > at_public_rates.prompt_caching
 
 
+@pytest.mark.parametrize(
+    "baseline_id, selected_id, selected_multiplier, billed_input, classifier_cost, expected",
+    [
+        ("baseline", "selected", 0.1, None, 0.0, 0.0135),
+        ("baseline", "selected", 2.0, None, 0.0, -0.015),
+        ("baseline", "selected", 1.0, None, 0.0, 0.0),
+        ("baseline", "selected", 0.1, 0.004, 0.001, 0.01),
+        ("baseline", "baseline", 0.1, 0.004, 0.001, -0.001),
+        (None, "selected", 0.1, None, 0.0, 0.0),
+        ("baseline", None, 0.1, None, 0.0, 0.0),
+        (None, None, 0.1, None, 0.0, 0.0),
+        ("", "selected", 0.1, None, 0.0, 0.0),
+        ("baseline", "", 0.1, None, 0.0, 0.0),
+    ],
+)
+def test_autorouter_savings_distinguishes_priced_deployments(
+    baseline_id: str | None,
+    selected_id: str | None,
+    selected_multiplier: float,
+    billed_input: float | None,
+    classifier_cost: float,
+    expected: float,
+) -> None:
+    router: Final = Router(
+        model_list=[
+            {
+                "model_name": name,
+                "litellm_params": {
+                    "model": "anthropic/claude-opus-5",
+                    "api_key": "test-key",
+                    "input_cost_per_token": 1e-5 * multiplier,
+                    "output_cost_per_token": 5e-5 * multiplier,
+                },
+                "model_info": {"id": name},
+            }
+            for name, multiplier in (("baseline", 1.0), ("selected", selected_multiplier))
+        ]
+    )
+    result: Final = compute_savings_spend(
+        model="claude-opus-5",
+        custom_llm_provider="anthropic",
+        compression_saved_tokens=0,
+        gateway_injected_cache=False,
+        model_id=selected_id,
+        llm_router=lambda: router,
+        routing_decision={
+            "savings_baseline_model": "anthropic/claude-opus-5",
+            "savings_baseline_deployment_id": baseline_id,
+            "conversation_continuing": False,
+            "classifier_cost": classifier_cost,
+        },
+        usage_object={"prompt_tokens": 1000, "completion_tokens": 100, "total_tokens": 1100},
+        cost_breakdown=None if billed_input is None else {"input_cost": billed_input, "output_cost": 0.0},
+    )
+    assert result.autorouter == pytest.approx(expected)
+
+
+@pytest.mark.parametrize("selected_model", ["azure/contract-deployment", "contract-deployment"])
+def test_autorouter_savings_recognizes_one_deployment_under_its_base_model(selected_model: str) -> None:
+    router: Final = Router(
+        model_list=[
+            {
+                "model_name": "contract",
+                "litellm_params": {
+                    "model": "azure/contract-deployment",
+                    "api_key": "test-key",
+                    "api_base": "https://example.openai.azure.com",
+                    "input_cost_per_token": 0.0001,
+                    "output_cost_per_token": 0.0002,
+                    "cache_read_input_token_cost": 0.00001,
+                },
+                "model_info": {"id": "contract", "base_model": "azure/gpt-5.5"},
+            }
+        ]
+    )
+    result: Final = compute_savings_spend(
+        model=selected_model,
+        custom_llm_provider="azure",
+        compression_saved_tokens=0,
+        gateway_injected_cache=False,
+        model_id="contract",
+        llm_router=lambda: router,
+        routing_decision={
+            "savings_baseline_model": "azure/gpt-5.5",
+            "savings_baseline_deployment_id": "contract",
+            "conversation_continuing": True,
+        },
+        usage_object={
+            "prompt_tokens": 21000,
+            "completion_tokens": 100,
+            "total_tokens": 21100,
+            "prompt_tokens_details": {"text_tokens": 1000, "cached_tokens": 0, "cache_creation_tokens": 20000},
+        },
+        cost_breakdown={"input_cost": 2.1, "output_cost": 0.02},
+    )
+    assert result.autorouter == 0.0
+
+
 def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
     """A hardest-tier deployment with a negotiated rate is what the traffic would
     really have cost; pricing its model publicly misstates the saving."""

From 9a9b4c4c2538bf0df6d68eaabe48cefbb4824e7d Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Mon, 7 Sep 2026 23:37:42 -0700
Subject: [PATCH 11/27] feat(ui): show auto-router classification rate (#40192)

---
 .../AutoRouterBenchmarksTab.test.tsx          | 12 ++++-----
 .../_components/AutoRouterBenchmarksTab.tsx   | 26 ++++++++++++++-----
 .../_components/costOptimizationUtils.test.ts | 18 +++++++++++++
 .../_components/costOptimizationUtils.ts      |  7 +++++
 ...KeyAutoRouterUsageTab.integration.test.tsx |  2 +-
 5 files changed, 52 insertions(+), 13 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx
index 006da4f2725..2820a9dce83 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx
@@ -187,10 +187,10 @@ describe("AutoRouterBenchmarksTab", () => {
   });
 
   it.each([
-    { spend: 20665.28, classifier_cost: 342.18, turns: 140815, llm: "$20,323.10", cost: "$342.18" },
-    { spend: 0, classifier_cost: 0, turns: 0, llm: "$0.00", cost: "$0.00" },
-    { spend: 0.002, classifier_cost: 0.0004, turns: 100, llm: "$0.0016", cost: "$0.0004" },
-  ])("shows total classification cost across $turns turns without a per-turn rate", ({ llm, cost, ...values }) => {
+    { spend: 20665.28, classifier_cost: 342.18, turns: 140815, llm: "$20,323.10", cost: "$342.18", rate: "$2.43" },
+    { spend: 0, classifier_cost: 0, turns: 0, llm: "$0.00", cost: "$0.00", rate: "$0.00" },
+    { spend: 0.002, classifier_cost: 0.0004, turns: 100, llm: "$0.0016", cost: "$0.0004", rate: "$0.0040" },
+  ])("shows total classification cost and its rate across $turns turns", ({ llm, cost, rate, ...values }) => {
     const stats = totals({ ...values, saved_spend: 10126.28, baseline_spend: values.spend + 10126.28 });
     mockHook({ data: response([group(stats)], stats) });
     renderTab();
@@ -201,7 +201,7 @@ describe("AutoRouterBenchmarksTab", () => {
         .map((node) => node.textContent)
         .slice(1, 3),
     ).toEqual([llm, cost]);
-    expect(screen.queryByText(/1K turns/)).not.toBeInTheDocument();
+    expect(screen.getByText(`(${rate} / 1K turns)`)).toBeInTheDocument();
     expect(screen.getAllByText("$10,126.28").length).toBeGreaterThan(0);
   });
 
@@ -237,7 +237,7 @@ describe("AutoRouterBenchmarksTab", () => {
     expect(terms).toEqual([
       "Actual auto-router spend",
       "LLM spend",
-      "Classification cost",
+      "Classification cost($2.00 / 1K turns)",
       "Estimated spend at highest-tier model",
     ]);
     expect(values).toEqual(["$359.86", "$353.71", "$6.15", "$2,534.45"]);
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx
index 33ad1bfe555..ce5ab1c6776 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx
@@ -30,7 +30,7 @@ import {
   type BenchmarkView,
   type BucketRow,
 } from "./autoRouterBenchmarks";
-import { formatRangeLabel, usd } from "./costOptimizationUtils";
+import { classificationRatePer1kTurns, formatRangeLabel, usd } from "./costOptimizationUtils";
 import ShadowEvalSection from "./ShadowEvalSection";
 import TierTurnsChart from "./TierTurnsChart";
 import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks";
@@ -52,9 +52,17 @@ const Metric: React.FC<{ label: string; value: string; hint?: string }> = ({ lab
   </Card>
 );
 
-const SpendRow: React.FC<{ label: string; value: string; subdued?: boolean }> = ({ label, value, subdued }) => (
+const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued?: boolean }> = ({
+  label,
+  value,
+  hint,
+  subdued,
+}) => (
   <dl className="flex flex-wrap items-baseline justify-between gap-x-6 gap-y-1 py-2">
-    <dt className="min-w-0 text-sm text-muted-foreground">{label}</dt>
+    <dt className="flex min-w-0 flex-wrap items-baseline gap-x-2 text-sm text-muted-foreground">
+      {label}
+      {hint && <span className="text-xs">{hint}</span>}
+    </dt>
     <dd
       className={`min-w-0 break-all tabular-nums ${subdued ? "text-sm font-normal text-muted-foreground" : "text-base font-semibold text-foreground"}`}
     >
@@ -99,6 +107,11 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {
               subdued
               label="Classification cost"
               value={stats.classifier_cost == null ? "Unavailable" : usd(stats.classifier_cost)}
+              hint={
+                stats.classifier_cost == null
+                  ? undefined
+                  : classificationRatePer1kTurns(stats.classifier_cost, stats.turns)
+              }
             />
           </div>
           {stats.classifier_cost == null && (
@@ -277,9 +290,10 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
       <p className="text-xs text-muted-foreground">
         Compares your actual routed spend with the estimated cost of using only the most expensive model configured in
         the auto-router. It accounts for both the cache savings from staying on one model and the added cache costs from
-        switching models. Savings are net of recorded LLM classification cost, which is included in actual spend. The
-        range counts whole sessions that overlap it, so totals can differ slightly from the Overall tab, which buckets
-        savings by UTC day.
+        switching models. Savings are net of recorded LLM classification cost, which is included in actual spend.
+        Classification cost per 1K turns is averaged over all auto-router turns, including those that skip
+        classification. The range counts whole sessions that overlap it, so totals can differ slightly from the Overall
+        tab, which buckets savings by UTC day.
       </p>
 
       <div className="space-y-4">
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts
index 9a2d7a0b0ec..5d2c48e6440 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts
@@ -7,6 +7,7 @@ import {
   SAVINGS_DRIVERS,
   SAVINGS_SERIES,
   buildDailyToolSeries,
+  classificationRatePer1kTurns,
   computeCacheLeakage,
   formatRangeLabel,
   isAnthropicModel,
@@ -401,6 +402,23 @@ describe("usd", () => {
   });
 });
 
+describe("classificationRatePer1kTurns", () => {
+  it("normalizes total classification cost to one thousand turns", () => {
+    expect(classificationRatePer1kTurns(342.18, 140815)).toBe("($2.43 / 1K turns)");
+    expect(classificationRatePer1kTurns(0.0004, 100)).toBe("($0.0040 / 1K turns)");
+  });
+
+  it("shows a floor instead of rounding a real cost down to zero", () => {
+    expect(classificationRatePer1kTurns(0.00001, 1000)).toBe("(<$0.0001 / 1K turns)");
+    expect(classificationRatePer1kTurns(0.0001, 1000)).toBe("($0.0001 / 1K turns)");
+  });
+
+  it("reports zero when there are no turns or no classification cost", () => {
+    expect(classificationRatePer1kTurns(0, 0)).toBe("($0.00 / 1K turns)");
+    expect(classificationRatePer1kTurns(0, 100)).toBe("($0.00 / 1K turns)");
+  });
+});
+
 describe("savings driver colours", () => {
   it("keeps a driver's colour when a driver above it is filtered out", () => {
     // Charts colour by position in the data they are given, and the donut is given
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
index 7019b0d3301..464c779aa2b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts
@@ -10,6 +10,13 @@ export const usd = (value: number): string => {
   return `${value < 0 ? "-" : ""}$${formatNumberWithCommas(magnitude, decimals)}`;
 };
 
+export const classificationRatePer1kTurns = (classifierCost: number, turns: number): string => {
+  if (turns <= 0) return `(${usd(0)} / 1K turns)`;
+  const rate = (classifierCost * 1000) / turns;
+  if (rate > 0 && rate < 0.0001) return "(<$0.0001 / 1K turns)";
+  return `(${usd(rate)} / 1K turns)`;
+};
+
 export const pct = (ratio: number): string => `${formatNumberWithCommas(ratio * 100, 1)}%`;
 
 export const shortDate = (iso: string): string =>
diff --git a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx
index be95c0e600d..1f6a67b27ac 100644
--- a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx
+++ b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx
@@ -89,7 +89,7 @@ describe("KeyAutoRouterUsageTab", () => {
     expect(screen.getByText("$1.00")).toBeInTheDocument();
     expect(screen.getByText("Classification cost")).toBeInTheDocument();
     expect(screen.getByText("$0.2500")).toBeInTheDocument();
-    expect(screen.queryByText(/1K turns/)).not.toBeInTheDocument();
+    expect(screen.getByText("($62.50 / 1K turns)")).toBeInTheDocument();
     expect(screen.getByText("Estimated spend at highest-tier model")).toBeInTheDocument();
     expect(screen.getByText("$10.00")).toBeInTheDocument();
     expect(screen.getByText("Auto-router prompt caching")).toBeInTheDocument();

From 1af7a403c66e037bec2e0ae6ea455a1c10b17b1b Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Mon, 7 Sep 2026 23:40:02 -0700
Subject: [PATCH 12/27] feat(mcp): start the named server's OAuth directly for
 a resource-scoped gateway flow (#39933)

An aggregate gateway DCR authorize whose RFC 8707 resource resolves to exactly one
gateway-managed oauth2 server sealed that server into the flow and then sent the browser to
the generic connect grid anyway, so the user had to find the server the client had already
named and click Connect.

The connect URL now carries only the flow handle. GET /authorize/flow classifies the sealed
flow as unscoped, interactive, M2M, or stale, and returns the matching state to the page.
Interactive flows require a live per-user vendor credential before minting and do not burn the
flow on an early submit. M2M flows use the gateway's configured service credential and finish
without an interactive OAuth trip. Stale flows fail closed instead of becoming unscoped.
The existing explicit Finish action and a new Cancel path preserve deliberate user intent.
---
 litellm/proxy/_experimental/mcp_server/db.py  |  21 +-
 .../mcp_server/discoverable_endpoints.py      |  69 +++--
 .../mcp_server/gateway_dcr_flow.py            | 187 +++++++++----
 litellm/proxy/_lazy_openapi_snapshot.json     |  40 +++
 .../mcp_server/test_discoverable_endpoints.py |  52 ++++
 .../mcp_server/test_gateway_dcr_flow.py       | 254 ++++++++++++++++--
 .../src/app/chat/integrations/page.tsx        |  30 +--
 .../src/app/connect/page.test.tsx             |  90 +------
 ui/litellm-dashboard/src/app/connect/page.tsx |  23 +-
 .../chat/ConnectFlowBanner.test.tsx           |  69 +++--
 .../src/components/chat/ConnectFlowBanner.tsx | 123 ++++++---
 .../chat/ConnectFlowSurface.test.tsx          | 118 ++++++++
 .../components/chat/ConnectFlowSurface.tsx    |  59 ++++
 .../src/components/chat/MCPAppsPanel.test.tsx |   9 +
 .../src/components/chat/MCPAppsPanel.tsx      |  33 ++-
 .../src/components/networking.tsx             |  18 ++
 .../src/lib/http/client.test.ts               |   9 +
 ui/litellm-dashboard/src/lib/http/client.ts   |   6 +-
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  48 ++++
 19 files changed, 950 insertions(+), 308 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.test.tsx
 create mode 100644 ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.tsx

diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py
index 082a90fdcfb..7379126983a 100644
--- a/litellm/proxy/_experimental/mcp_server/db.py
+++ b/litellm/proxy/_experimental/mcp_server/db.py
@@ -4,7 +4,7 @@ import hashlib
 import json
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -125,6 +125,9 @@ class OAuthCredentialPayload(_OAuthCredentialAccessToken, total=False):
     server_id: str
 
 
+OAuthGrantState = Literal["valid", "refreshable", "absent"]
+
+
 class _OAuthTokenRefreshResponse(TypedDict, total=False):
     access_token: str
     refresh_token: str
@@ -1465,6 +1468,15 @@ def is_oauth_credential_expired(cred: OAuthCredentialPayload, buffer_seconds: in
         return False
 
 
+def oauth_grant_state(cred: OAuthCredentialPayload | None) -> OAuthGrantState:
+    """Classify local grant readiness without attempting a refresh or checking upstream revocation."""
+    if not cred or not cred.get("access_token"):
+        return "absent"
+    if not is_oauth_credential_expired(cred, buffer_seconds=MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS):
+        return "valid"
+    return "refreshable" if cred.get("refresh_token") else "absent"
+
+
 async def get_user_oauth_credential(
     prisma_client: PrismaClient,
     user_id: str,
@@ -1727,12 +1739,11 @@ async def resolve_valid_user_oauth_token(
     dict it already holds. ``prisma_client`` is fetched lazily and only when a refresh
     actually happens, so the valid-token path never requires a DB handle.
     """
-    if not cred or not cred.get("access_token"):
+    grant: Final = oauth_grant_state(cred)
+    if cred is None or grant == "absent":
         return None
-    if not is_oauth_credential_expired(cred, buffer_seconds=MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS):
+    if grant == "valid":
         return cred
-    if not cred.get("refresh_token"):
-        return None
     if prisma_client is None:
         from litellm.proxy.utils import get_prisma_client_or_throw
 
diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
index e10bfd41ed6..cab4b6c161a 100644
--- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
+++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
@@ -43,9 +43,11 @@ from litellm.proxy._experimental.mcp_server.faults import (
     render_token_fault,
 )
 from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
+    VendorCredentialState,
     aggregate_authorize,
     aggregate_token,
     complete_connect_flow,
+    describe_connect_flow,
     introspect_gateway_token,
     is_gateway_dcr_client_id,
     is_proxy_api_resource,
@@ -798,21 +800,7 @@ def _bridge_access_denied_redirect(redirect_uri: str, state: str, mcp_server: MC
     return RedirectResponse(_append_query_params(redirect_uri, params), status_code=302)
 
 
-async def _bridge_authorize_access_denial(
-    litellm_user_id: str,
-    mcp_server: MCPServer,
-    redirect_uri: str,
-    state: str,
-) -> RedirectResponse | None:
-    """The denial redirect for a signed-in user who cannot reach the target server, or None to proceed.
-
-    Admits the user exactly as MCP egress will (the same ``reload_admitted_user`` constructor and the
-    same ``get_allowed_mcp_servers`` resolver), so an envelope is minted only when the resulting
-    session can actually list and call the server's tools. Without this gate the flow completes, the
-    client shows connected, and every tool request fail-closes to an empty list with nothing telling
-    the operator why. An availability fault (5xx, e.g. a DB outage's 503) propagates; an unknown or
-    deactivated user denies like a missing grant, fail closed.
-    """
+async def _user_can_reach_mcp_server(user_id: str, server_id: str) -> bool:
     from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
         MCPRequestHandler,
     )
@@ -821,13 +809,22 @@ async def _bridge_authorize_access_denial(
     )
 
     try:
-        admitted: Final = await MCPRequestHandler.reload_admitted_user(litellm_user_id)
+        admitted: Final = await MCPRequestHandler.reload_admitted_user(user_id)
     except HTTPException as exc:
         if exc.status_code >= 500:
             raise
-        return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
-    allowed_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers(admitted)
-    if mcp_server.server_id in allowed_server_ids:
+        return False
+    return server_id in await global_mcp_server_manager.get_allowed_mcp_servers(admitted)
+
+
+async def _bridge_authorize_access_denial(
+    litellm_user_id: str,
+    mcp_server: MCPServer,
+    redirect_uri: str,
+    state: str,
+) -> RedirectResponse | None:
+    """The denial redirect for a signed-in user who cannot reach the target server, or None to proceed."""
+    if await _user_can_reach_mcp_server(litellm_user_id, mcp_server.server_id):
         return None
     return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
 
@@ -1910,6 +1907,38 @@ async def token_endpoint(
     )
 
 
+async def _vendor_credential_state(user_id: str, server_id: str) -> VendorCredentialState:
+    """Whether the gateway itself can see a live vendor credential for this user and server.
+
+    The one reading of "authorized" the connect page displays and the finish step enforces, so
+    the button a user sees and the grant they get cannot disagree. A read fault is neither, and
+    fails the scoped grant closed."""
+    from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415  # circular import at module load
+        get_user_oauth_credential,
+        oauth_grant_state,
+    )
+    from litellm.proxy.proxy_server import prisma_client  # noqa: PLC0415  # circular import at module load
+
+    if prisma_client is None:
+        return "unavailable"
+    try:
+        credential: Final = await get_user_oauth_credential(prisma_client, user_id, server_id)
+    except Exception:  # noqa: BLE001  # a credential-read fault must fail the scoped grant closed
+        return "unavailable"
+    return "absent" if oauth_grant_state(credential) == "absent" else "present"
+
+
+@router.get("/authorize/flow")
+async def authorize_flow(request: Request, flow: str) -> Response:
+    return await describe_connect_flow(
+        request=request,
+        flow_handle=flow,
+        session_user_id=_session_cookie_user_id(request),
+        lookup_vendor_credential=_vendor_credential_state,
+        lookup_server_reachability=_user_can_reach_mcp_server,
+    )
+
+
 @router.post("/authorize/complete")
 async def authorize_complete(
     request: Request,
@@ -1934,6 +1963,8 @@ async def authorize_complete(
         delivery=delivery,
         team_id=team_id,
         decision=decision,
+        lookup_vendor_credential=_vendor_credential_state,
+        lookup_server_reachability=_user_can_reach_mcp_server,
     )
 
 
diff --git a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py
index c7b0045dde5..3d94fa345d0 100644
--- a/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py
+++ b/litellm/proxy/_experimental/mcp_server/gateway_dcr_flow.py
@@ -152,10 +152,8 @@ _AUTH_CODE_DEBUG_KEY: Final = "gateway_authorization_code"
 
 ReloadUserFailure = Literal["unresolvable", "unavailable", "faulted", "no_active_key"]
 ReloadUser = Callable[[str], Awaitable[ReloadUserFailure | None]]
-"""Injected live-user revalidation (the token endpoint's mirror of admission):
-``None`` means the user is active; ``unavailable`` is a retryable DB outage; ``faulted`` is
-a DB fault retrying will not clear (still 503, worded so nobody just waits); anything else
-fails the grant closed."""
+VendorCredentialState = Literal["present", "absent", "unavailable"]
+"""The per-user vendor credential read has three outcomes: present, absent, or unavailable."""
 
 _DB_UNAVAILABLE_DESCRIPTION: Final = "the gateway database is unavailable; retry"
 _DB_FAULTED_DESCRIPTION: Final = (
@@ -195,6 +193,16 @@ class ConsentTeam(BaseModel):
     team_alias: str | None = None
 
 
+class LookupVendorCredential(Protocol):
+    """Injected read of a user's vendor credential for one server."""
+
+    def __call__(self, user_id: str, server_id: str, /) -> Awaitable[VendorCredentialState]: ...
+
+
+class LookupServerReachability(Protocol):
+    def __call__(self, user_id: str, server_id: str, /) -> Awaitable[bool]: ...
+
+
 class LookupConsentTeams(Protocol):
     """Injected lookup of the teams a signed-in user may bind a proxy-API credential to."""
 
@@ -205,6 +213,14 @@ async def _refuse_proxy_credential(user_id: str, team_id: str | None) -> ProxyCr
     return "unresolvable"
 
 
+async def _unavailable_vendor_credential(user_id: str, server_id: str) -> VendorCredentialState:
+    return "unavailable"
+
+
+async def _unreachable_server(user_id: str, server_id: str) -> bool:
+    return False
+
+
 class GatewayDcrClient(BaseModel):
     """The registration record sealed into a gateway DCR ``client_id``.
 
@@ -449,7 +465,10 @@ def aggregate_authorize(
 
     A per-server RFC 8707 ``resource`` naming a gateway-managed oauth2 server scopes the
     flow to that one server: the scope is sealed into the flow, carried into the code, and
-    bound into the session token, while the connect page interlude runs exactly as before.
+    bound into the session token. The connect URL carries only the flow handle; the page
+    learns the client origin, the scoped server, and whether its vendor OAuth is done from
+    :func:`describe_connect_flow`, which reads the sealed flow, so nothing a link can carry
+    steers which server the page authorizes or names on the confirmation.
 
     Validation failures respond directly with 400 and never redirect: per RFC 6749
     section 4.1.2.1 an unvalidated redirect URI must not receive an error redirect, and
@@ -474,10 +493,7 @@ def aggregate_authorize(
         resource_server_id=scoped_server.server_id if scoped_server is not None else None,
         audience=None,
     )
-    connect_url: Final = _append_query_params(
-        f"{base_url}/ui/connect",
-        (("connect_flow", handle), ("connect_client", _origin_only(redirect_uri))),
-    )
+    connect_url: Final = _append_query_params(f"{base_url}/ui/connect", (("connect_flow", handle),))
     response: Final = RedirectResponse(connect_url, status_code=303)
     _set_flow_cookie(response, request, handle, flow)
     return response
@@ -684,6 +700,99 @@ def _origin_only(url: str) -> str:
     return f"{parsed.scheme}://{parsed.netloc}" if parsed.netloc else ""
 
 
+def _open_flow_for(
+    request: Request, flow_handle: str, session_user_id: str | None, now: datetime
+) -> _ConnectFlow | Response:
+    sealed_flow: Final = request.cookies.get(_flow_cookie_name(flow_handle))
+    if sealed_flow is None:
+        return _oauth_error(400, "invalid_request", "unknown or expired connect flow")
+    flow: Final = _open_sealed(sealed_flow, _UNPREFIXED, _ConnectFlow, _CONNECT_FLOW_DEBUG_KEY)
+    if flow is None or now.timestamp() >= flow.exp:
+        return _oauth_error(400, "invalid_request", "unknown or expired connect flow")
+    if session_user_id is None:
+        return _oauth_error(401, "login_required", "sign in to LiteLLM to finish connecting")
+    if session_user_id != flow.user_id:
+        return _oauth_error(403, "access_denied", "the signed-in user does not match this connect flow")
+    return flow
+
+
+async def _flow_target(
+    flow: _ConnectFlow, lookup_server_reachability: LookupServerReachability
+) -> tuple[Literal["unscoped", "interactive", "m2m", "stale"], MCPServer | None]:
+    if flow.resource_server_id is None:
+        return "unscoped", None
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (  # noqa: PLC0415  # import cycle
+        MCPServerManager,
+        global_mcp_server_manager,
+    )
+
+    server: Final = global_mcp_server_manager.get_mcp_server_by_id(flow.resource_server_id)
+    if (
+        server is None
+        or not server.is_gateway_managed_oauth2
+        or not await lookup_server_reachability(flow.user_id, server.server_id)
+    ):
+        return "stale", None
+    state: Final = "m2m" if MCPServerManager.effective_oauth2_flow(server) == "client_credentials" else "interactive"
+    return state, server
+
+
+class ConnectFlowDescription(TypedDict):
+    """What the connect page is allowed to know about one in-flight flow."""
+
+    state: ReadOnly[Literal["unscoped", "interactive", "m2m", "stale"]]
+    client_origin: ReadOnly[str]
+    server_id: ReadOnly[str | None]
+    server_name: ReadOnly[str | None]
+    connected: ReadOnly[bool | None]
+
+
+async def _describe_opened_flow(
+    flow: _ConnectFlow,
+    lookup_vendor_credential: LookupVendorCredential,
+    lookup_server_reachability: LookupServerReachability,
+) -> ConnectFlowDescription | Response:
+    state, server = await _flow_target(flow, lookup_server_reachability)
+    if state == "interactive" and server is not None:
+        credential: Final = await lookup_vendor_credential(flow.user_id, server.server_id)
+        if credential == "unavailable":
+            return _oauth_error(503, "temporarily_unavailable", _DB_UNAVAILABLE_DESCRIPTION)
+        interactive_description: Final[ConnectFlowDescription] = {
+            "state": state,
+            "client_origin": _origin_only(flow.redirect_uri),
+            "server_id": server.server_id,
+            "server_name": server.server_name or server.alias or server.name,
+            "connected": credential == "present",
+        }
+        return interactive_description
+    described: Final[ConnectFlowDescription] = {
+        "state": state,
+        "client_origin": _origin_only(flow.redirect_uri),
+        "server_id": None if server is None else server.server_id,
+        "server_name": None if server is None else (server.server_name or server.alias or server.name),
+        "connected": state == "m2m" or None,
+    }
+    return described
+
+
+async def describe_connect_flow(
+    request: Request,
+    flow_handle: str,
+    session_user_id: str | None,
+    lookup_vendor_credential: LookupVendorCredential,
+    lookup_server_reachability: LookupServerReachability,
+) -> Response:
+    opened: Final = _open_flow_for(request, flow_handle, session_user_id, datetime.now(timezone.utc))
+    if isinstance(opened, Response):
+        return opened
+    described: Final = await _describe_opened_flow(opened, lookup_vendor_credential, lookup_server_reachability)
+    return (
+        described
+        if isinstance(described, Response)
+        else JSONResponse(content=described, headers=TOKEN_NO_CACHE_HEADERS)
+    )
+
+
 async def complete_connect_flow(
     request: Request,
     flow_handle: str,
@@ -692,56 +801,34 @@ async def complete_connect_flow(
     delivery: str | None = None,
     team_id: str | None = None,
     decision: str | None = None,
+    lookup_vendor_credential: LookupVendorCredential = _unavailable_vendor_credential,
+    lookup_server_reachability: LookupServerReachability = _unreachable_server,
 ) -> Response:
-    """The deliberate finish step of the connect flow: mint the gateway authorization
-    code and send the browser back to the client.
+    """Mint the code only after a deliberate POST by the sealed user.
 
-    Reached by POST so a cross-site GET cannot trigger it, and bound to the HttpOnly
-    per-flow cookie plus an exact match between the signed-in user and the user sealed
-    into the flow: a link crafted by another party dies here with ``access_denied``
-    instead of minting a code for the victim's identity. The flow is single-use (an atomic
-    claim on its ``jti``), so a double-submit cannot mint two codes from one sign-in.
-
-    ``delivery`` chooses how the code reaches the client. Default (absent or
-    ``"redirect"``) is the 303 to the client's registered redirect URI. ``"manual"``
-    renders the callback URL on a page instead, for a client whose redirect URI is a
-    loopback host but which runs on a DIFFERENT machine than the browser (EC2/SSH box,
-    container): the 303 would dereference the browser machine's loopback and the code
-    would never arrive, so the user carries it over by pasting the URL into the client or
-    fetching it from the client machine's terminal. Manual delivery is honored only for
-    loopback redirect URIs; a routable redirect URI works from any browser by
-    construction, so those flows always redirect. The user who sees the page is exactly
-    the user the 303 would have carried the code to, and the same user already sees the
-    code today in the dead redirect's address bar, so the page exposes the code to no new
-    party. Unknown ``delivery`` values are rejected rather than defaulted: a client that
-    asked for manual delivery and got a dead redirect instead would silently lose its
-    code.
-
-    ``decision`` and ``team_id`` come from the native-client consent page. ``"deny"``
-    burns the flow and sends the client ``error=access_denied`` so it stops waiting;
-    ``team_id`` is sealed into the code only for proxy-API flows, where it picks which of
-    the user's teams the minted credential is attributed to.
+    A scoped flow additionally requires its sealed server to have a live vendor credential
+    before a code can be minted. The check happens before the single-use claim, so a
+    premature submit can be retried after authorization; denial deliberately bypasses it.
     """
     if delivery not in (None, "redirect", "manual"):
         return _oauth_error(400, "invalid_request", "delivery must be 'redirect' or 'manual'")
     if decision not in (None, "approve", "deny"):
         return _oauth_error(400, "invalid_request", "decision must be 'approve' or 'deny'")
-    sealed_flow: Final = request.cookies.get(_flow_cookie_name(flow_handle))
-    if sealed_flow is None:
-        return _oauth_error(400, "invalid_request", "unknown or expired connect flow")
-    flow: Final = _open_sealed(sealed_flow, _UNPREFIXED, _ConnectFlow, _CONNECT_FLOW_DEBUG_KEY)
-    if flow is None:
-        return _oauth_error(400, "invalid_request", "unknown or expired connect flow")
     now: Final = datetime.now(timezone.utc)
-    if now.timestamp() >= flow.exp:
-        return _oauth_error(400, "invalid_request", "the connect flow has expired; restart the connection")
-    if session_user_id is None:
-        return _oauth_error(401, "login_required", "sign in to LiteLLM to finish connecting")
-    if session_user_id != flow.user_id:
-        return _oauth_error(403, "access_denied", "the signed-in user does not match this connect flow")
+    opened: Final = _open_flow_for(request, flow_handle, session_user_id, now)
+    if isinstance(opened, Response):
+        return opened
+    if decision != "deny":
+        described: Final = await _describe_opened_flow(opened, lookup_vendor_credential, lookup_server_reachability)
+        if isinstance(described, Response):
+            return described
+        if described["state"] == "stale":
+            return _oauth_error(400, "invalid_request", "the requested MCP server is no longer available")
+        if described["connected"] is False:
+            return _oauth_error(400, "invalid_request", "authorize the requested MCP server before finishing")
     flow_refusal: Final = _claim_refusal(
         await _SingleUseGuard(cache).claim(
-            f"{_USED_FLOW_CACHE_PREFIX}{flow.jti}", CONNECT_FLOW_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
+            f"{_USED_FLOW_CACHE_PREFIX}{opened.jti}", CONNECT_FLOW_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
         ),
         replayed=_oauth_error(
             400, "invalid_request", "this connect flow was already completed; restart the connection"
@@ -750,7 +837,7 @@ async def complete_connect_flow(
     if flow_refusal is not None:
         return flow_refusal
     response: Final = (
-        _denied_flow_response(flow) if decision == "deny" else _approved_flow_response(flow, delivery, team_id, now)
+        _denied_flow_response(opened) if decision == "deny" else _approved_flow_response(opened, delivery, team_id, now)
     )
     path, secure = _cookie_path_and_secure(request)
     response.delete_cookie(key=_flow_cookie_name(flow_handle), path=path, secure=secure, httponly=True, samesite="lax")
diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json
index dba2b2428aa..71475320c2c 100644
--- a/litellm/proxy/_lazy_openapi_snapshot.json
+++ b/litellm/proxy/_lazy_openapi_snapshot.json
@@ -19886,6 +19886,46 @@
           ]
         }
       },
+      "/authorize/flow": {
+        "get": {
+          "operationId": "authorize_flow_authorize_flow_get",
+          "parameters": [
+            {
+              "in": "query",
+              "name": "flow",
+              "required": true,
+              "schema": {
+                "title": "Flow",
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {}
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "summary": "Authorize Flow",
+          "tags": [
+            "mcp_discoverable"
+          ]
+        }
+      },
       "/callback": {
         "get": {
           "description": "OAuth 2.0 authorization response handler for MCP loopback clients.\n\nAccepts either:\n\n- A successful authorization response (``code`` + ``state``), which is\n  forwarded back to the validated client ``redirect_uri`` with the\n  original (un-wrapped) ``state``.\n- An error response (``error``[+``error_description``/``error_uri``]), per\n  RFC 6749 \u00a74.1.2.1. When ``state`` is present and decodes to a trusted\n  ``redirect_uri``, the error params are propagated back to the client so\n  its OAuth library can surface them. Otherwise we render an HTML error\n  page so the user is not left on an opaque 422 / blank screen.",
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
index be4206a1faf..763200c3709 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
@@ -4,6 +4,7 @@ import hashlib
 import json
 import time
 from base64 import urlsafe_b64encode
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -18,6 +19,57 @@ if TYPE_CHECKING:
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
 
+def _stored_grant(access_token="access-token", refresh_token=None, expires_in_seconds=None, expires_at=None):
+    credential = {"type": "oauth2", "access_token": access_token}
+    if refresh_token is not None:
+        credential["refresh_token"] = refresh_token
+    if expires_in_seconds is not None:
+        credential["expires_at"] = (datetime.now(timezone.utc) + timedelta(seconds=expires_in_seconds)).isoformat()
+    if expires_at is not None:
+        credential["expires_at"] = expires_at
+    return credential
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("fields", "egress_has_token"),
+    [
+        (None, False),
+        ({"access_token": "", "refresh_token": "refresh-token"}, False),
+        ({}, True),
+        ({"expires_at": "never"}, True),
+        ({"expires_in_seconds": 600}, True),
+        ({"expires_in_seconds": 30}, False),
+        ({"expires_in_seconds": -300}, False),
+        ({"expires_in_seconds": -300, "refresh_token": ""}, False),
+        ({"expires_in_seconds": 30, "refresh_token": "refresh-token"}, True),
+        ({"expires_in_seconds": -300, "refresh_token": "refresh-token"}, True),
+    ],
+)
+async def test_vendor_credential_state_agrees_with_egress_token_resolution(monkeypatch, fields, egress_has_token):
+    from litellm.proxy._experimental.mcp_server import db as mcp_db
+    from litellm.proxy._experimental.mcp_server import discoverable_endpoints
+
+    monkeypatch.setattr(mcp_db, "MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", 60)
+    credential = _stored_grant(**fields) if fields is not None else None
+    read = AsyncMock(return_value=credential)
+    refresh = AsyncMock(return_value=_stored_grant(access_token="fresh-token", expires_in_seconds=3600))
+    prisma = MagicMock()
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma)
+    monkeypatch.setattr(mcp_db, "get_user_oauth_credential", read)
+    monkeypatch.setattr(mcp_db, "refresh_user_oauth_token", refresh)
+
+    connect = await discoverable_endpoints._vendor_credential_state("user-1", "server-1")
+    read.assert_awaited_once_with(prisma, "user-1", "server-1")
+    refresh.assert_not_awaited()
+    egress = await mcp_db.resolve_valid_user_oauth_token(
+        user_id="user-1", server=MagicMock(), cred=credential, prisma_client=prisma
+    )
+
+    assert (egress is not None) is egress_has_token
+    assert connect == ("present" if egress_has_token else "absent")
+
+
 # Fixture to mock IP address check for all MCP tests
 # This prevents tests from failing due to IP-based access control
 @pytest.fixture(autouse=True)
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py
index 1670370f082..73a52a8d2e8 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_gateway_dcr_flow.py
@@ -26,6 +26,7 @@ from litellm.proxy._experimental.mcp_server.gateway_dcr_flow import (
     aggregate_authorize,
     aggregate_token,
     complete_connect_flow,
+    describe_connect_flow,
     introspect_gateway_token,
     is_gateway_dcr_client_id,
     is_proxy_api_resource,
@@ -230,7 +231,7 @@ async def test_authorize_with_session_hands_browser_to_connect_page_with_flow_co
     assert location.path == "/ui/connect"
     params = parse_qs(location.query)
     handle = params["connect_flow"][0]
-    assert params["connect_client"] == ["https://claude.ai"]
+    assert set(params) == {"connect_flow"}
     set_cookie = response.headers["set-cookie"]
     assert f"{CONNECT_FLOW_COOKIE_PREFIX}{handle}" in set_cookie
     assert "HttpOnly" in set_cookie
@@ -889,14 +890,60 @@ def _opened_principal(payload):
     return admitted.principal
 
 
-async def _finish_connect_page(response):
+class _VendorCredential:
+    def __init__(self, state="present"):
+        self.calls = []
+        self.state = state
+
+    async def __call__(self, user_id, server_id):
+        self.calls.append((user_id, server_id))
+        return self.state
+
+
+class _ServerReachability:
+    def __init__(self, reachable=True):
+        self.calls = []
+        self.reachable = reachable
+
+    async def __call__(self, user_id, server_id):
+        self.calls.append((user_id, server_id))
+        return self.reachable
+
+
+async def _complete_page(response, scoped_server=None, vendor=None, reachable=None, cache=None, **overrides):
+    from unittest.mock import patch
+
     handle, cookies = _flow_cookie_from(response)
-    completed = await complete_connect_flow(
-        request=_request("/authorize/complete", cookies=cookies, method="POST"),
-        flow_handle=handle,
-        session_user_id="u1",
-        cache=DualCache(),
-    )
+    with patch(_MANAGER_PATCH) as manager:
+        manager.get_mcp_server_by_id.return_value = scoped_server
+        return await complete_connect_flow(
+            request=_request("/authorize/complete", cookies=cookies, method="POST"),
+            flow_handle=handle,
+            session_user_id="u1",
+            cache=cache or DualCache(),
+            lookup_vendor_credential=vendor or _VendorCredential(),
+            lookup_server_reachability=reachable or _ServerReachability(),
+            **overrides,
+        )
+
+
+async def _describe_page(response, scoped_server=None, vendor=None, reachable=None, session_user_id="u1", cookies=None):
+    from unittest.mock import patch
+
+    handle, flow_cookies = _flow_cookie_from(response)
+    with patch(_MANAGER_PATCH) as manager:
+        manager.get_mcp_server_by_id.return_value = scoped_server
+        return await describe_connect_flow(
+            request=_request("/authorize/flow", cookies=flow_cookies if cookies is None else cookies),
+            flow_handle=handle,
+            session_user_id=session_user_id,
+            lookup_vendor_credential=vendor or _VendorCredential(),
+            lookup_server_reachability=reachable or _ServerReachability(),
+        )
+
+
+async def _finish_connect_page(response, scoped_server=None):
+    completed = await _complete_page(response, scoped_server=scoped_server)
     return parse_qs(urlparse(completed.headers["location"]).query)["code"][0]
 
 
@@ -910,23 +957,43 @@ def _sealed_wire_json(sealed, prefix, debug_key):
 
 @pytest.mark.asyncio
 async def test_scoped_authorize_runs_connect_page_with_sealed_scope():
-    """LIT-4917: a per-server RFC 8707 resource naming a gateway-managed oauth2 server
-    seals that server into the flow. The connect page interlude runs exactly as before
-    (the scope restricts, it never skips consent), and the code minted at the finish step
-    and the session pair it redeems for are both scoped."""
+    """LIT-4917 plus LIT-7075: a per-server RFC 8707 resource naming a gateway-managed oauth2
+    server seals that server into the flow. The connect URL carries only the handle; the page
+    learns the scoped server and its vendor state from describe_connect_flow, and the finish
+    step refuses to mint a scoped code until that vendor credential exists, without burning
+    the flow. The code minted afterwards and the session pair it redeems for are both scoped."""
     from unittest.mock import patch
 
     client_id = (await _register([REDIRECT_URI]))["client_id"]
+    github = _scoped_mcp_server()
     with patch(_MANAGER_PATCH) as manager:
-        manager.get_mcp_server_by_name.return_value = _scoped_mcp_server()
+        manager.get_mcp_server_by_name.return_value = github
         response = _scoped_authorize(client_id, SCOPED_RESOURCE)
     assert response.status_code == 303
-    assert "/ui/connect" in response.headers["location"]
+    location = urlparse(response.headers["location"])
+    assert location.path == "/ui/connect"
+    assert set(parse_qs(location.query)) == {"connect_flow"}
     _, cookies = _flow_cookie_from(response)
     assert (
         _sealed_wire_json(next(iter(cookies.values())), "", "gateway_connect_flow")["resource_server_id"] == "github-id"
     )
-    code = await _finish_connect_page(response)
+    described = await _describe_page(response, scoped_server=github, vendor=_VendorCredential("absent"))
+    assert json.loads(described.body) == {
+        "state": "interactive",
+        "client_origin": "https://claude.ai",
+        "server_id": "github-id",
+        "server_name": "github",
+        "connected": False,
+    }
+    cache = DualCache()
+    premature = await _complete_page(response, scoped_server=github, vendor=_VendorCredential("absent"), cache=cache)
+    assert premature.status_code == 400
+    assert "authorize the requested MCP server" in json.loads(premature.body)["error_description"]
+    present = _VendorCredential("present")
+    completed = await _complete_page(response, scoped_server=github, vendor=present, cache=cache)
+    assert completed.status_code == 303
+    assert present.calls == [("u1", "github-id")]
+    code = parse_qs(urlparse(completed.headers["location"]).query)["code"][0]
     assert (
         _sealed_wire_json(code, GATEWAY_AUTH_CODE_PREFIX, "gateway_authorization_code")["resource_server_id"]
         == "github-id"
@@ -952,9 +1019,11 @@ async def test_scoped_authorize_runs_connect_page_with_sealed_scope():
 )
 async def test_unscoped_resources_leave_flow_and_token_byte_identical(resource, resolves):
     """Every resource shape outside 'exactly one gateway-managed server' keeps today's flow:
-    connect page interlude, and NONE of the minted artifacts carry the scope key on the
-    wire, not the flow cookie, not the code, not the session JWT, so an unscoped flow
-    started on a new pod completes on a pod whose strict models predate the claim."""
+    the generic connect grid (describe names no server, the finish step never consults the
+    vendor credential), and NONE of the minted
+    artifacts carry the scope key on the wire, not the flow cookie, not the code, not the
+    session JWT, so an unscoped flow started on a new pod completes on a pod whose strict
+    models predate the claim."""
     import base64
     from unittest.mock import patch
 
@@ -963,10 +1032,18 @@ async def test_unscoped_resources_leave_flow_and_token_byte_identical(resource,
         manager.get_mcp_server_by_name.return_value = None if resolves is None else _scoped_mcp_server()
         response = _scoped_authorize(client_id, resource)
     assert response.status_code == 303
-    assert "/ui/connect" in response.headers["location"]
+    location = urlparse(response.headers["location"])
+    assert location.path == "/ui/connect"
+    assert set(parse_qs(location.query)) == {"connect_flow"}
     _, cookies = _flow_cookie_from(response)
     assert "resource_server_id" not in _sealed_wire_json(next(iter(cookies.values())), "", "gateway_connect_flow")
-    code = await _finish_connect_page(response)
+    vendor = _VendorCredential("absent")
+    described = await _describe_page(response, vendor=vendor)
+    assert json.loads(described.body)["state"] == "unscoped"
+    assert json.loads(described.body)["server_id"] is None
+    completed = await _complete_page(response, vendor=vendor)
+    assert vendor.calls == []
+    code = parse_qs(urlparse(completed.headers["location"]).query)["code"][0]
     assert "resource_server_id" not in _sealed_wire_json(code, GATEWAY_AUTH_CODE_PREFIX, "gateway_authorization_code")
     token_response = await _redeem(code, client_id)
     payload = json.loads(token_response.body)
@@ -979,19 +1056,150 @@ async def test_unscoped_resources_leave_flow_and_token_byte_identical(resource,
 @pytest.mark.asyncio
 async def test_scoped_authorize_delegate_server_stays_unscoped():
     """A delegate-auth oauth2 server is outside the gateway-managed set (its keyless flow is
-    upstream PKCE via the relay), so a resource naming it never scopes the gateway flow."""
+    upstream PKCE via the relay), so a resource naming it never scopes the gateway flow and
+    never narrows the connect page to it."""
     from unittest.mock import patch
 
     client_id = (await _register([REDIRECT_URI]))["client_id"]
     with patch(_MANAGER_PATCH) as manager:
         manager.get_mcp_server_by_name.return_value = _scoped_mcp_server(delegate_auth_to_upstream=True)
         response = _scoped_authorize(client_id, SCOPED_RESOURCE)
-    assert "/ui/connect" in response.headers["location"]
+    location = urlparse(response.headers["location"])
+    assert location.path == "/ui/connect"
+    assert set(parse_qs(location.query)) == {"connect_flow"}
+    assert json.loads((await _describe_page(response)).body)["server_id"] is None
     code = await _finish_connect_page(response)
     token_response = await _redeem(code, client_id)
     assert _opened_principal(json.loads(token_response.body)).resource_server_id is None
 
 
+@pytest.mark.asyncio
+async def test_m2m_scoped_flow_mints_without_a_user_credential():
+    """A client-credentials server is already authorized by its gateway service credential, so
+    a resource-scoped flow finishes without consulting the per-user vault."""
+    from unittest.mock import patch
+
+    client_id = (await _register([REDIRECT_URI]))["client_id"]
+    m2m = _scoped_mcp_server(oauth2_flow="client_credentials")
+    with patch(_MANAGER_PATCH) as manager:
+        manager.get_mcp_server_by_name.return_value = m2m
+        response = _scoped_authorize(client_id, SCOPED_RESOURCE)
+    vendor = _VendorCredential("unavailable")
+    described = await _describe_page(response, scoped_server=m2m, vendor=vendor)
+    assert json.loads(described.body)["state"] == "m2m"
+    assert json.loads(described.body)["connected"] is True
+    assert vendor.calls == []
+    completed = await _complete_page(response, scoped_server=m2m, vendor=vendor)
+    assert completed.status_code == 303
+    assert vendor.calls == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("oauth2_flow", ["authorization_code", "client_credentials"])
+async def test_unreachable_scoped_flow_cannot_describe_or_finish(oauth2_flow):
+    from unittest.mock import patch
+
+    client_id = (await _register([REDIRECT_URI]))["client_id"]
+    server = _scoped_mcp_server(oauth2_flow=oauth2_flow)
+    with patch(_MANAGER_PATCH) as manager:
+        manager.get_mcp_server_by_name.return_value = server
+        response = _scoped_authorize(client_id, SCOPED_RESOURCE)
+    reachable = _ServerReachability(False)
+    vendor = _VendorCredential("present")
+    described = await _describe_page(response, scoped_server=server, reachable=reachable, vendor=vendor)
+    assert json.loads(described.body) == {
+        "state": "stale",
+        "client_origin": "https://claude.ai",
+        "server_id": None,
+        "server_name": None,
+        "connected": None,
+    }
+    cache = DualCache()
+    refused = await _complete_page(response, scoped_server=server, reachable=reachable, vendor=vendor, cache=cache)
+    assert refused.status_code == 400
+    assert vendor.calls == []
+    assert reachable.calls == [("u1", "github-id"), ("u1", "github-id")]
+    completed = await _complete_page(response, scoped_server=server, vendor=vendor, cache=cache)
+    assert completed.status_code == 303
+
+
+@pytest.mark.asyncio
+async def test_stale_scoped_flow_remains_distinct_from_unscoped():
+    """A server removed after authorize stays a stale scoped flow, so the page cannot offer a
+    broader unscoped grant or report a misleading Finish action."""
+    from unittest.mock import patch
+
+    client_id = (await _register([REDIRECT_URI]))["client_id"]
+    with patch(_MANAGER_PATCH) as manager:
+        manager.get_mcp_server_by_name.return_value = _scoped_mcp_server()
+        response = _scoped_authorize(client_id, SCOPED_RESOURCE)
+    described = await _describe_page(response, scoped_server=None)
+    assert json.loads(described.body)["state"] == "stale"
+    assert json.loads(described.body)["connected"] is None
+    stale = await _complete_page(response, scoped_server=None)
+    assert stale.status_code == 400
+    assert json.loads(stale.body)["error_description"] == "the requested MCP server is no longer available"
+
+
+@pytest.mark.asyncio
+async def test_scoped_flow_deny_and_stale_server_never_need_the_vendor_credential():
+    """Cancel is the escape hatch: a scoped user who cannot finish the vendor step still ends
+    the flow with access_denied and no credential lookup. A scoped server that is no longer
+    gateway-managed refuses to mint (nothing could serve that code) but also burns nothing."""
+    from unittest.mock import patch
+
+    client_id = (await _register([REDIRECT_URI]))["client_id"]
+    github = _scoped_mcp_server()
+    with patch(_MANAGER_PATCH) as manager:
+        manager.get_mcp_server_by_name.return_value = github
+        response = _scoped_authorize(client_id, SCOPED_RESOURCE)
+    cache = DualCache()
+    skipped_reachability = _ServerReachability(False)
+    stale = await _complete_page(response, scoped_server=None, reachable=skipped_reachability, cache=cache)
+    assert stale.status_code == 400
+    assert skipped_reachability.calls == []
+    assert json.loads(stale.body)["error_description"] == "the requested MCP server is no longer available"
+    described = await _describe_page(response, scoped_server=github, vendor=_VendorCredential("unavailable"))
+    assert described.status_code == 503
+    vendor = _VendorCredential("absent")
+    deny_reachability = _ServerReachability(False)
+    denied = await _complete_page(
+        response,
+        scoped_server=github,
+        vendor=vendor,
+        reachable=deny_reachability,
+        cache=cache,
+        decision="deny",
+    )
+    assert denied.status_code == 303
+    assert parse_qs(urlparse(denied.headers["location"]).query)["error"] == ["access_denied"]
+    assert vendor.calls == []
+    assert deny_reachability.calls == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "session_user_id, cookies, expected_status, expected_error",
+    [
+        ("u1", {}, 400, "invalid_request"),
+        ("u1", {"mcp_connect_flow_wrong": "garbage"}, 400, "invalid_request"),
+        (None, None, 401, "login_required"),
+        ("u2", None, 403, "access_denied"),
+    ],
+)
+async def test_describe_connect_flow_refuses_exactly_like_the_finish_step(
+    session_user_id, cookies, expected_status, expected_error
+):
+    """The page's read of the flow is gated the same way minting is: the HttpOnly cookie for
+    that handle must open and the signed-in user must be the sealed one. A lure link with a
+    made-up handle therefore learns nothing and starts nothing."""
+    client_id = (await _register([REDIRECT_URI]))["client_id"]
+    response = _authorize(client_id, session_user_id="u1")
+    described = await _describe_page(response, session_user_id=session_user_id, cookies=cookies)
+    assert described.status_code == expected_status
+    assert json.loads(described.body)["error"] == expected_error
+
+
 @pytest.mark.asyncio
 async def test_token_rejects_resource_conflicting_with_sealed_scope():
     """RFC 8707 section 2.2: redeeming a scoped code (or rotating a scoped refresh token)
@@ -1005,7 +1213,7 @@ async def test_token_rejects_resource_conflicting_with_sealed_scope():
     with patch(_MANAGER_PATCH) as manager:
         manager.get_mcp_server_by_name.return_value = github
         response = _scoped_authorize(client_id, SCOPED_RESOURCE)
-    code = await _finish_connect_page(response)
+    code = await _finish_connect_page(response, scoped_server=github)
 
     with patch(_MANAGER_PATCH) as manager:
         manager.get_mcp_server_by_name.return_value = linear
diff --git a/ui/litellm-dashboard/src/app/chat/integrations/page.tsx b/ui/litellm-dashboard/src/app/chat/integrations/page.tsx
index 30ce62d8081..a663e16c2b2 100644
--- a/ui/litellm-dashboard/src/app/chat/integrations/page.tsx
+++ b/ui/litellm-dashboard/src/app/chat/integrations/page.tsx
@@ -1,43 +1,19 @@
 "use client";
 
-import { Suspense, useEffect } from "react";
-import { useRouter, useSearchParams } from "next/navigation";
+import { Suspense } from "react";
 import { useChatShell } from "@/contexts/ChatShellContext";
-import MCPAppsPanel from "@/components/chat/MCPAppsPanel";
-import ConnectFlowBanner from "@/components/chat/ConnectFlowBanner";
+import ConnectFlowSurface from "@/components/chat/ConnectFlowSurface";
 
 // useSearchParams() requires a Suspense boundary for static export.
 function IntegrationsPageContent() {
   const { accessToken, selectedMCPServers, setSelectedMCPServers } = useChatShell();
-  const router = useRouter();
-  const searchParams = useSearchParams();
-  const oauthReturn = searchParams.get("mcpOauthReturn");
-  // Set by the gateway DCR authorize when a DCR client sends the user here to
-  // authorize servers before finishing sign-in (see gateway_dcr_flow.py). The
-  // handle keys the sealed per-flow cookie; connect_client is the client origin
-  // for display only. connect_flow is NOT cleaned from the URL: the finish form
-  // needs it, and the sealed cookie (not the URL) is the security boundary.
-  const connectFlow = searchParams.get("connect_flow");
-  const connectClient = searchParams.get("connect_client");
-
-  // Clean up the OAuth return param after it's been consumed — real routing means
-  // we no longer need it to pick a tab, but it should not linger in the address bar.
-  useEffect(() => {
-    if (oauthReturn) {
-      const url = new URL(window.location.href);
-      url.searchParams.delete("mcpOauthReturn");
-      router.replace(url.pathname + url.search);
-    }
-  }, [oauthReturn, router]);
 
   return (
     <div className="flex-1 min-h-0 overflow-auto w-full py-8 px-8">
-      {connectFlow && <ConnectFlowBanner flowHandle={connectFlow} clientOrigin={connectClient} />}
-      <MCPAppsPanel
+      <ConnectFlowSurface
         accessToken={accessToken}
         selectedServers={selectedMCPServers}
         onChange={setSelectedMCPServers}
-        connectMode={!!connectFlow}
       />
     </div>
   );
diff --git a/ui/litellm-dashboard/src/app/connect/page.test.tsx b/ui/litellm-dashboard/src/app/connect/page.test.tsx
index 7d49a8b6a4c..6a6ba24bd87 100644
--- a/ui/litellm-dashboard/src/app/connect/page.test.tsx
+++ b/ui/litellm-dashboard/src/app/connect/page.test.tsx
@@ -2,101 +2,29 @@ import { afterEach, describe, expect, it, vi } from "vitest";
 import { render, screen } from "@testing-library/react";
 import ConnectPage from "./page";
 
-interface PanelProps {
+interface SurfaceProps {
   accessToken: string;
   selectedServers: string[];
   onChange: (servers: string[]) => void;
-  connectMode?: boolean;
 }
 
-interface BannerProps {
-  flowHandle: string;
-  clientOrigin: string | null;
-}
-
-const { mockReplace, mockPanel, mockBanner, state } = vi.hoisted(() => {
-  const state = {
-    oauthReturn: null as string | null,
-    connectFlow: null as string | null,
-    connectClient: null as string | null,
-  };
-  return {
-    state,
-    mockReplace: vi.fn(),
-    mockPanel: vi.fn((_props: PanelProps) => <div data-testid="mcp-apps-panel" />),
-    mockBanner: vi.fn((_props: BannerProps) => <div data-testid="connect-flow-banner" />),
-  };
-});
-
-vi.mock("next/navigation", () => ({
-  useRouter: () => ({ replace: mockReplace }),
-  useSearchParams: () => ({
-    get: (key: string) => {
-      if (key === "mcpOauthReturn") return state.oauthReturn;
-      if (key === "connect_flow") return state.connectFlow;
-      if (key === "connect_client") return state.connectClient;
-      return null;
-    },
-  }),
+const { mockSurface } = vi.hoisted(() => ({
+  mockSurface: vi.fn((_props: SurfaceProps) => <div data-testid="connect-flow-surface" />),
 }));
+
 vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({
   default: () => ({ accessToken: "token-123" }),
 }));
-vi.mock("@/components/chat/MCPAppsPanel", () => ({ default: mockPanel }));
-vi.mock("@/components/chat/ConnectFlowBanner", () => ({ default: mockBanner }));
+vi.mock("@/components/chat/ConnectFlowSurface", () => ({ default: mockSurface }));
 
 describe("ConnectPage", () => {
   afterEach(() => {
-    state.oauthReturn = null;
-    state.connectFlow = null;
-    state.connectClient = null;
-    mockReplace.mockClear();
-    mockPanel.mockClear();
-    mockBanner.mockClear();
+    mockSurface.mockClear();
   });
 
-  it("renders the MCP connect panel with the user's access token", () => {
+  it("renders the gateway connect surface with the user's access token and an empty selection", () => {
     render(<ConnectPage />);
-    expect(screen.getByTestId("mcp-apps-panel")).toBeInTheDocument();
-    expect(mockPanel.mock.calls[0][0]).toMatchObject({ accessToken: "token-123", selectedServers: [] });
-  });
-
-  it("strips the mcpOauthReturn param from the URL after an OAuth return", () => {
-    state.oauthReturn = "apps";
-    window.history.replaceState({}, "", "/connect?mcpOauthReturn=apps");
-    render(<ConnectPage />);
-    expect(mockReplace).toHaveBeenCalledWith("/connect");
-  });
-
-  it("does not rewrite the URL when there is no OAuth return param", () => {
-    render(<ConnectPage />);
-    expect(mockReplace).not.toHaveBeenCalled();
-  });
-
-  it("mounts the gateway connect banner and puts the panel in connect mode for a DCR flow", () => {
-    state.connectFlow = "flow-handle-123";
-    state.connectClient = "https://claude.ai";
-    render(<ConnectPage />);
-    expect(screen.getByTestId("connect-flow-banner")).toBeInTheDocument();
-    expect(mockBanner.mock.calls[0][0]).toMatchObject({
-      flowHandle: "flow-handle-123",
-      clientOrigin: "https://claude.ai",
-    });
-    expect(mockPanel.mock.calls[0][0].connectMode).toBe(true);
-  });
-
-  it("shows no connect banner and leaves connect mode off for a plain visit", () => {
-    render(<ConnectPage />);
-    expect(screen.queryByTestId("connect-flow-banner")).not.toBeInTheDocument();
-    expect(mockBanner).not.toHaveBeenCalled();
-    expect(mockPanel.mock.calls[0][0].connectMode).toBe(false);
-  });
-
-  it("keeps the connect flow handle in the URL while stripping the OAuth return param", () => {
-    state.oauthReturn = "apps";
-    state.connectFlow = "flow-handle-123";
-    window.history.replaceState({}, "", "/connect?connect_flow=flow-handle-123&mcpOauthReturn=apps");
-    render(<ConnectPage />);
-    expect(mockReplace).toHaveBeenCalledWith("/connect?connect_flow=flow-handle-123");
+    expect(screen.getByTestId("connect-flow-surface")).toBeInTheDocument();
+    expect(mockSurface.mock.calls[0][0]).toMatchObject({ accessToken: "token-123", selectedServers: [] });
   });
 });
diff --git a/ui/litellm-dashboard/src/app/connect/page.tsx b/ui/litellm-dashboard/src/app/connect/page.tsx
index 3f0c269e86b..652c044197b 100644
--- a/ui/litellm-dashboard/src/app/connect/page.tsx
+++ b/ui/litellm-dashboard/src/app/connect/page.tsx
@@ -1,36 +1,19 @@
 "use client";
 
-import { Suspense, useEffect, useState } from "react";
-import { useRouter, useSearchParams } from "next/navigation";
+import { Suspense, useState } from "react";
 import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
-import MCPAppsPanel from "@/components/chat/MCPAppsPanel";
-import ConnectFlowBanner from "@/components/chat/ConnectFlowBanner";
+import ConnectFlowSurface from "@/components/chat/ConnectFlowSurface";
 
 function ConnectPageContent() {
   const { accessToken } = useAuthorized();
   const [selectedServers, setSelectedServers] = useState<string[]>([]);
-  const router = useRouter();
-  const searchParams = useSearchParams();
-  const oauthReturn = searchParams.get("mcpOauthReturn");
-  const connectFlow = searchParams.get("connect_flow");
-  const connectClient = searchParams.get("connect_client");
-
-  useEffect(() => {
-    if (oauthReturn) {
-      const url = new URL(window.location.href);
-      url.searchParams.delete("mcpOauthReturn");
-      router.replace(url.pathname + url.search);
-    }
-  }, [oauthReturn, router]);
 
   return (
     <div className="mx-auto w-full max-w-5xl px-8 py-8">
-      {connectFlow && <ConnectFlowBanner flowHandle={connectFlow} clientOrigin={connectClient} />}
-      <MCPAppsPanel
+      <ConnectFlowSurface
         accessToken={accessToken ?? ""}
         selectedServers={selectedServers}
         onChange={setSelectedServers}
-        connectMode={!!connectFlow}
       />
     </div>
   );
diff --git a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx
index b3cd6e229af..5caf15d1fce 100644
--- a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx
+++ b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.test.tsx
@@ -1,59 +1,61 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { render, screen } from "@testing-library/react";
+import type { ConnectFlowStatus } from "@/components/networking";
 import ConnectFlowBanner, { isLoopbackOrigin } from "./ConnectFlowBanner";
 
 vi.mock("@/components/networking", () => ({
   getProxyBaseUrl: () => "https://gateway.example.com",
 }));
 
+vi.mock("@/hooks/useUserMcpOAuthFlow", () => ({
+  useUserMcpOAuthFlow: () => ({ startOAuthFlow: vi.fn(), status: "idle" }),
+}));
+
 afterEach(() => {
   vi.restoreAllMocks();
-  sessionStorage.clear();
 });
 
+const unscoped = (client_origin: string): ConnectFlowStatus => ({
+  state: "unscoped",
+  client_origin,
+  server_id: null,
+  server_name: null,
+  connected: null,
+});
+
+const renderBanner = (clientOrigin: string) =>
+  render(
+    <ConnectFlowBanner
+      flowHandle="flow-handle-123"
+      flow={unscoped(clientOrigin)}
+      accessToken="tok"
+      onConnected={vi.fn()}
+      failed={false}
+    />,
+  );
+
 describe("ConnectFlowBanner", () => {
-  it("posts the flow handle to the proxy /authorize/complete as a full-page form", () => {
-    const { container } = render(<ConnectFlowBanner flowHandle="flow-handle-123" clientOrigin="https://claude.ai" />);
+  it("posts only the flow handle to the proxy /authorize/complete as a full-page form", () => {
+    const { container } = renderBanner("https://claude.ai");
 
     const form = container.querySelector("form")!;
     expect(form).toHaveAttribute("method", "POST");
     expect(form).toHaveAttribute("action", "https://gateway.example.com/authorize/complete");
-
-    const hidden = form.querySelector('input[name="flow"]') as HTMLInputElement;
-    expect(hidden.value).toBe("flow-handle-123");
-    // No token, code, or secret is ever placed in the form; the sealed cookie carries them.
+    expect(screen.getByDisplayValue("flow-handle-123")).toHaveAttribute("name", "flow");
     expect(form.innerHTML).not.toContain("token");
-  });
-
-  it("shows the client origin so the user knows what they are connecting to", () => {
-    render(<ConnectFlowBanner flowHandle="h" clientOrigin="https://claude.ai" />);
-    expect(screen.getAllByText(/claude\.ai/).length).toBeGreaterThan(0);
     expect(screen.getByRole("button", { name: /finish connecting/i })).toBeInTheDocument();
+    expect(screen.queryByRole("button", { name: "Cancel" })).not.toBeInTheDocument();
   });
 
-  it("falls back to a generic label when the client origin is unknown", () => {
-    render(<ConnectFlowBanner flowHandle="h" clientOrigin={null} />);
-    expect(screen.getAllByText(/the application/).length).toBeGreaterThan(0);
-  });
-
-  it("offers manual delivery for a loopback client, posted only when checked", () => {
-    const { container } = render(
-      <ConnectFlowBanner flowHandle="flow-handle-123" clientOrigin="http://localhost:3118" />,
-    );
-
-    const checkbox = container.querySelector('input[type="checkbox"][name="delivery"]') as HTMLInputElement;
-    expect(checkbox).not.toBeNull();
+  it("offers manual delivery only for a loopback client, posted only when checked", () => {
+    const loopback = renderBanner("http://localhost:3118");
+    const checkbox = loopback.container.querySelector('input[type="checkbox"][name="delivery"]') as HTMLInputElement;
     expect(checkbox.value).toBe("manual");
     expect(checkbox.checked).toBe(false);
-    expect(screen.getByText(/remote or SSH machine/i)).toBeInTheDocument();
-  });
+    loopback.unmount();
 
-  it("does not offer manual delivery for a routable client origin or an unknown one", () => {
-    const routable = render(<ConnectFlowBanner flowHandle="h" clientOrigin="https://claude.ai" />);
+    const routable = renderBanner("https://claude.ai");
     expect(routable.container.querySelector('input[name="delivery"]')).toBeNull();
-
-    const unknown = render(<ConnectFlowBanner flowHandle="h" clientOrigin={null} />);
-    expect(unknown.container.querySelector('input[name="delivery"]')).toBeNull();
   });
 
   it("classifies loopback origins like the server does", () => {
@@ -70,12 +72,9 @@ describe("ConnectFlowBanner", () => {
   });
 
   it("does NOT complete the flow on pagehide (completion requires the explicit button)", () => {
-    // Security regression: an attacker could lure a signed-in victim to their own client's
-    // authorize URL; the victim merely closing the tab must NOT deliver a victim-bound code.
-    // Completion is a deliberate button press, never a side effect of leaving the page.
     const beaconMock = vi.fn(() => true);
     vi.stubGlobal("navigator", { ...navigator, sendBeacon: beaconMock });
-    render(<ConnectFlowBanner flowHandle="flow-xyz" clientOrigin="https://claude.ai" />);
+    renderBanner("https://claude.ai");
 
     window.dispatchEvent(new Event("pagehide"));
 
diff --git a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx
index cea42f916f8..0d6e708f734 100644
--- a/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx
+++ b/ui/litellm-dashboard/src/components/chat/ConnectFlowBanner.tsx
@@ -2,30 +2,18 @@
 
 import React from "react";
 import { CheckCircle } from "lucide-react";
-import { getProxyBaseUrl } from "@/components/networking";
+import { getProxyBaseUrl, ConnectFlowStatus } from "@/components/networking";
+import { OAuth2ConnectButton } from "@/components/chat/MCPAppsPanel";
 
 interface Props {
   flowHandle: string;
-  clientOrigin: string | null;
+  flow?: ConnectFlowStatus;
+  accessToken: string;
+  onConnected: () => void;
+  failed: boolean;
 }
 
-/**
- * The interlude shown when a DCR client (Claude Desktop, MCP Inspector) sends the user
- * through the gateway sign-in and lands them on the apps grid to authorize servers. The
- * grid below authorizes individual servers into the per-user vault; this banner is the
- * finish step that returns the user to the client.
- *
- * Finishing requires the explicit "Finish connecting" button: a native form POST to the proxy's
- * /authorize/complete, which mints the gateway authorization code and 303-redirects to the DCR
- * client's own redirect URI (the full-page navigation carries the HttpOnly per-flow cookie and
- * follows the cross-origin redirect to the client's loopback).
- *
- * The button press IS the consent gate and must not be bypassed. An earlier version auto-finished
- * on tab close via navigator.sendBeacon; that let an attacker who lured a signed-in victim to their
- * own client's authorize URL harvest a victim-bound code the moment the victim closed the tab
- * (no click). Merely visiting the authorize URL is attacker-inducible, so completion has to be a
- * deliberate user action, not a side effect of leaving the page.
- */
+/** Finish remains an explicit POST because a cross-site navigation must never mint a code. */
 export function isLoopbackOrigin(origin: string | null): boolean {
   if (!origin) return false;
   try {
@@ -36,10 +24,44 @@ export function isLoopbackOrigin(origin: string | null): boolean {
   }
 }
 
-const ConnectFlowBanner: React.FC<Props> = ({ flowHandle, clientOrigin }) => {
+const copyFor = (flow: ConnectFlowStatus | undefined, failed: boolean): readonly [string, string] => {
+  const clientLabel = flow?.client_origin ?? "the application";
+  const serverLabel = flow?.server_name ?? "the requested MCP server";
+  if (failed || flow === undefined || flow.state === "stale") {
+    return [
+      "The connection cannot continue",
+      `The gateway could not validate this connection. Cancel to return to ${clientLabel}.`,
+    ];
+  }
+  if (flow.state === "unscoped") {
+    return [
+      `Connect your MCP servers to ${clientLabel}`,
+      `Authorize the servers you want to use below, then click Finish connecting to return to ${clientLabel}.`,
+    ];
+  }
+  if (flow.state === "interactive" && !flow.connected) {
+    return [
+      `Allow ${clientLabel} to use ${serverLabel}`,
+      `Authorize ${serverLabel} below to continue, or cancel to send ${clientLabel} away.`,
+    ];
+  }
+  return [
+    `Allow ${clientLabel} to use ${serverLabel}`,
+    `Click Finish connecting to give ${clientLabel} access to ${serverLabel} as you.`,
+  ];
+};
+
+const ConnectFlowBanner: React.FC<Props> = ({ flowHandle, flow, accessToken, onConnected, failed }) => {
   const action = `${getProxyBaseUrl()}/authorize/complete`;
-  const clientLabel = clientOrigin ?? "the application";
-  const loopbackClient = isLoopbackOrigin(clientOrigin);
+  const state = failed || flow === undefined ? "stale" : flow.state;
+  const canFinish = state === "unscoped" || (state !== "stale" && flow?.connected === true);
+  const canCancel = state !== "unscoped";
+  const loopbackClient = isLoopbackOrigin(flow?.client_origin ?? null);
+  const vendorServer =
+    state === "interactive" && flow?.connected === false && flow.server_id !== null
+      ? { server_id: flow.server_id, server_name: flow.server_name }
+      : null;
+  const copy = copyFor(flow, failed);
 
   return (
     <div className="mb-6 rounded-lg border border-primary/30 bg-primary/5 px-5 py-4">
@@ -47,27 +69,48 @@ const ConnectFlowBanner: React.FC<Props> = ({ flowHandle, clientOrigin }) => {
         <div className="flex items-start gap-3 min-w-0">
           <CheckCircle className="h-5 w-5 text-primary shrink-0 mt-0.5" />
           <div className="min-w-0">
-            <p className="text-sm font-semibold text-foreground">Connect your MCP servers to {clientLabel}</p>
-            <p className="text-[13px] text-muted-foreground mt-0.5">
-              Authorize the servers you want to use below, then click Finish connecting to return to {clientLabel}.
-            </p>
+            <p className="text-sm font-semibold text-foreground">{copy[0]}</p>
+            <p className="text-[13px] text-muted-foreground mt-0.5">{copy[1]}</p>
           </div>
         </div>
-        <form method="POST" action={action} className="shrink-0">
-          <input type="hidden" name="flow" value={flowHandle} />
-          <button
-            type="submit"
-            className="h-[38px] rounded-md bg-primary px-4 text-sm font-semibold text-primary-foreground hover:bg-primary/90"
-          >
-            Finish connecting
-          </button>
-          {loopbackClient && (
-            <label className="mt-2 flex items-center gap-2 text-[13px] text-muted-foreground">
-              <input type="checkbox" name="delivery" value="manual" />
-              My client is on a remote or SSH machine
-            </label>
+        <div className="flex shrink-0 gap-2">
+          {vendorServer !== null && (
+            <OAuth2ConnectButton
+              server={vendorServer}
+              accessToken={accessToken}
+              onConnect={onConnected}
+              variant="button"
+              autoStartKey={`litellm-mcp-autostart:${flowHandle}`}
+            />
           )}
-        </form>
+          <form method="POST" action={action}>
+            <input type="hidden" name="flow" value={flowHandle} />
+            {canFinish && (
+              <button
+                type="submit"
+                className="h-[38px] rounded-md bg-primary px-4 text-sm font-semibold text-primary-foreground hover:bg-primary/90"
+              >
+                Finish connecting
+              </button>
+            )}
+            {canCancel && (
+              <button
+                type="submit"
+                name="decision"
+                value="deny"
+                className="ml-2 h-[38px] rounded-md border px-4 text-sm font-semibold text-foreground hover:bg-accent/40"
+              >
+                Cancel
+              </button>
+            )}
+            {loopbackClient && (
+              <label className="mt-2 flex items-center gap-2 text-[13px] text-muted-foreground">
+                <input type="checkbox" name="delivery" value="manual" />
+                My client is on a remote or SSH machine
+              </label>
+            )}
+          </form>
+        </div>
       </div>
     </div>
   );
diff --git a/ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.test.tsx b/ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.test.tsx
new file mode 100644
index 00000000000..cf59dcfc368
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.test.tsx
@@ -0,0 +1,118 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { act, render, screen, waitFor } from "@testing-library/react";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import ConnectFlowSurface from "./ConnectFlowSurface";
+import { fetchConnectFlow } from "@/components/networking";
+
+const { startOAuthFlow, state, onSuccess } = vi.hoisted(() => ({
+  startOAuthFlow: vi.fn(),
+  onSuccess: { current: undefined as (() => void) | undefined },
+  state: { oauthReturn: null as string | null, connectFlow: null as string | null },
+}));
+
+vi.mock("next/navigation", () => ({
+  useRouter: () => ({ replace: vi.fn() }),
+  useSearchParams: () => ({
+    get: (key: string) => ({ mcpOauthReturn: state.oauthReturn, connect_flow: state.connectFlow })[key] ?? null,
+  }),
+}));
+vi.mock("@/components/networking", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("@/components/networking")>()),
+  fetchConnectFlow: vi.fn(),
+  getProxyBaseUrl: () => "https://gateway.example.com",
+}));
+vi.mock("@/components/chat/MCPAppsPanel", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("@/components/chat/MCPAppsPanel")>()),
+  default: () => <div data-testid="mcp-apps-panel" />,
+}));
+vi.mock("@/hooks/useUserMcpOAuthFlow", () => ({
+  useUserMcpOAuthFlow: ({ onSuccess: success }: { onSuccess: () => void }) => {
+    onSuccess.current = success;
+    return { startOAuthFlow, status: "idle" };
+  },
+}));
+
+const flow = (state: "unscoped" | "interactive" | "m2m" | "stale", connected: boolean | null = null) => ({
+  state,
+  client_origin: "https://claude.ai",
+  server_id: state === "interactive" || state === "m2m" ? "s-design" : null,
+  server_name: state === "interactive" || state === "m2m" ? "design_tool" : null,
+  connected,
+});
+
+const renderSurface = () =>
+  render(
+    <QueryClientProvider client={new QueryClient({ defaultOptions: { queries: { retry: false } } })}>
+      <ConnectFlowSurface accessToken="token-123" selectedServers={[]} onChange={vi.fn()} />
+    </QueryClientProvider>,
+  );
+
+afterEach(() => {
+  state.oauthReturn = null;
+  state.connectFlow = null;
+  onSuccess.current = undefined;
+  sessionStorage.clear();
+  vi.clearAllMocks();
+});
+
+describe("ConnectFlowSurface", () => {
+  it.each([
+    { result: flow("unscoped"), grid: true, finish: true, cancel: false, oauthStarts: 0 },
+    { result: flow("interactive", false), grid: false, finish: false, cancel: true, oauthStarts: 1 },
+    { result: flow("interactive", true), grid: false, finish: true, cancel: true, oauthStarts: 0 },
+    { result: flow("m2m", true), grid: false, finish: true, cancel: true, oauthStarts: 0 },
+    { result: flow("stale"), grid: false, finish: false, cancel: true, oauthStarts: 0 },
+  ])(
+    "renders $result.state without widening its action surface",
+    async ({ result, grid, finish, cancel, oauthStarts }) => {
+      state.connectFlow = "flow-handle-123";
+      vi.mocked(fetchConnectFlow).mockResolvedValue(result);
+      renderSurface();
+
+      await screen.findByRole("button", { name: /finish connecting|cancel|connect/i });
+      await waitFor(() => expect(startOAuthFlow).toHaveBeenCalledTimes(oauthStarts));
+      expect(screen.queryByTestId("mcp-apps-panel") !== null).toBe(grid);
+      expect(screen.queryByRole("button", { name: /finish connecting/i }) !== null).toBe(finish);
+      expect(screen.queryByRole("button", { name: "Cancel" }) !== null).toBe(cancel);
+    },
+  );
+
+  it("keeps the grid and Finish hidden until the gateway accepts a handle", () => {
+    state.connectFlow = "flow-handle-123";
+    vi.mocked(fetchConnectFlow).mockReturnValue(new Promise(() => {}));
+    renderSurface();
+
+    expect(screen.queryByTestId("mcp-apps-panel")).not.toBeInTheDocument();
+    expect(screen.queryByRole("button", { name: /finish connecting/i })).not.toBeInTheDocument();
+    expect(screen.getByRole("button", { name: "Cancel" })).toHaveAttribute("value", "deny");
+  });
+
+  it("keeps the grid and Finish hidden when flow validation fails", async () => {
+    state.connectFlow = "invalid-handle";
+    vi.mocked(fetchConnectFlow).mockRejectedValue(new Error("invalid flow"));
+    renderSurface();
+
+    await screen.findByRole("button", { name: "Cancel" });
+    expect(screen.queryByTestId("mcp-apps-panel")).not.toBeInTheDocument();
+    expect(screen.queryByRole("button", { name: /finish connecting/i })).not.toBeInTheDocument();
+  });
+
+  it("refetches the sealed flow after the vendor connection completes", async () => {
+    state.connectFlow = "flow-handle-123";
+    vi.mocked(fetchConnectFlow)
+      .mockResolvedValueOnce(flow("interactive", false))
+      .mockResolvedValueOnce(flow("interactive", true));
+    renderSurface();
+
+    await waitFor(() => expect(startOAuthFlow).toHaveBeenCalledOnce());
+    await act(async () => onSuccess.current?.());
+
+    await screen.findByRole("button", { name: /finish connecting/i });
+  });
+
+  it("renders the ordinary panel without a flow handle", () => {
+    renderSurface();
+    expect(fetchConnectFlow).not.toHaveBeenCalled();
+    expect(screen.getByTestId("mcp-apps-panel")).toBeInTheDocument();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.tsx b/ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.tsx
new file mode 100644
index 00000000000..33a61fceacf
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/chat/ConnectFlowSurface.tsx
@@ -0,0 +1,59 @@
+"use client";
+
+import React, { useEffect } from "react";
+import { useRouter, useSearchParams } from "next/navigation";
+import { useQuery } from "@tanstack/react-query";
+import MCPAppsPanel from "@/components/chat/MCPAppsPanel";
+import ConnectFlowBanner from "@/components/chat/ConnectFlowBanner";
+import { fetchConnectFlow } from "@/components/networking";
+
+interface Props {
+  accessToken: string;
+  selectedServers: string[];
+  onChange: (servers: string[]) => void;
+}
+
+/** Renders the sealed gateway connect flow without trusting URL context. */
+const ConnectFlowSurface: React.FC<Props> = ({ accessToken, selectedServers, onChange }) => {
+  const router = useRouter();
+  const searchParams = useSearchParams();
+  const oauthReturn = searchParams.get("mcpOauthReturn");
+  const connectFlow = searchParams.get("connect_flow");
+
+  useEffect(() => {
+    if (oauthReturn) {
+      const url = new URL(window.location.href);
+      url.searchParams.delete("mcpOauthReturn");
+      router.replace(url.pathname + url.search);
+    }
+  }, [oauthReturn, router]);
+
+  const flowQuery = {
+    queryKey: ["gateway-connect-flow", connectFlow],
+    queryFn: () => fetchConnectFlow(connectFlow!),
+    enabled: !!connectFlow,
+    retry: false,
+  };
+  const { data: flow, isError, refetch } = useQuery(flowQuery);
+
+  if (connectFlow === null) {
+    return <MCPAppsPanel accessToken={accessToken} selectedServers={selectedServers} onChange={onChange} />;
+  }
+
+  return (
+    <>
+      <ConnectFlowBanner
+        flowHandle={connectFlow}
+        flow={flow}
+        accessToken={accessToken}
+        onConnected={refetch}
+        failed={isError}
+      />
+      {flow?.state === "unscoped" && (
+        <MCPAppsPanel accessToken={accessToken} selectedServers={selectedServers} onChange={onChange} connectMode />
+      )}
+    </>
+  );
+};
+
+export default ConnectFlowSurface;
diff --git a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.test.tsx b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.test.tsx
index e8795c36bc6..c9609405676 100644
--- a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.test.tsx
+++ b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.test.tsx
@@ -88,6 +88,13 @@ describe("MCPAppsPanel logos", () => {
 });
 
 const connectServers = [
+  {
+    server_id: "s-m2m",
+    server_name: "service_tool",
+    auth_type: "oauth2",
+    oauth2_flow: "client_credentials",
+    connected_app_reachable: true,
+  },
   {
     server_id: "s-reach",
     server_name: "reachable_srv",
@@ -124,6 +131,8 @@ describe("MCPAppsPanel connected-app reachability (LIT-4861)", () => {
     expect(vi.mocked(fetchMCPServers)).toHaveBeenCalledWith("tok", undefined, true);
     expect(screen.queryByText("unreachable_srv")).not.toBeInTheDocument();
     expect(screen.getByText("Connected (1)")).toBeInTheDocument();
+    expect(screen.getByText("service_tool")).toBeInTheDocument();
+    expect(screen.queryByText("Connect", { exact: true })).not.toBeInTheDocument();
     const toolCountFetchedIds = vi.mocked(listMCPTools).mock.calls.map((call) => call[1]);
     expect(toolCountFetchedIds).toContain("s-reach");
     expect(toolCountFetchedIds).not.toContain("s-unreach");
diff --git a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx
index 38a095c067d..1fee8923e94 100644
--- a/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx
+++ b/ui/litellm-dashboard/src/components/chat/MCPAppsPanel.tsx
@@ -13,23 +13,32 @@ import {
   getMCPOAuthUserCredentialStatus,
   listMCPTools,
 } from "../networking";
-import { AUTH_TYPE, MCPServer, MCPTool, handleTransport, isUnsupportedOnGatewayConnect } from "../mcp_tools/types";
+import {
+  getMcpOAuthMode,
+  MCPServer,
+  MCPTool,
+  handleTransport,
+  isUnsupportedOnGatewayConnect,
+} from "../mcp_tools/types";
 import { Logo } from "@/components/molecules/logo/Logo";
 import { toast } from "@/lib/toast";
 import { useUserMcpOAuthFlow } from "@/hooks/useUserMcpOAuthFlow";
+import { getSecureItem, setSecureItem } from "@/utils/secureStorage";
 
 interface OAuth2ConnectButtonProps {
-  server: MCPServer;
+  server: Pick<MCPServer, "server_id" | "server_name" | "alias">;
   accessToken: string;
   onConnect: (serverId: string) => void;
   variant?: "badge" | "button";
+  autoStartKey?: string | null;
 }
 
-const OAuth2ConnectButton: React.FC<OAuth2ConnectButtonProps> = ({
+export const OAuth2ConnectButton: React.FC<OAuth2ConnectButtonProps> = ({
   server,
   accessToken,
   onConnect,
   variant = "badge",
+  autoStartKey = null,
 }) => {
   const name = server.server_name ?? server.alias ?? server.server_id;
   const { startOAuthFlow, status } = useUserMcpOAuthFlow({
@@ -39,6 +48,12 @@ const OAuth2ConnectButton: React.FC<OAuth2ConnectButtonProps> = ({
     onSuccess: useCallback(() => onConnect(server.server_id), [onConnect, server.server_id]),
   });
 
+  useEffect(() => {
+    if (autoStartKey === null || status !== "idle" || getSecureItem(autoStartKey) !== null) return;
+    setSecureItem(autoStartKey, "1");
+    startOAuthFlow();
+  }, [autoStartKey, status, startOAuthFlow]);
+
   const loading = status === "authorizing" || status === "exchanging";
 
   if (variant === "button") {
@@ -190,7 +205,7 @@ const MCPAppsPanel: React.FC<Props> = ({ accessToken, selectedServers, onChange,
         if (!isCurrentLoad()) return;
         const list: MCPServer[] = Array.isArray(serverData) ? serverData : serverData?.data ?? [];
         const reachable = connectMode ? list.filter((s) => s.connected_app_reachable !== false) : list;
-        const oauthServers = reachable.filter((s) => s.auth_type === AUTH_TYPE.OAUTH2);
+        const oauthServers = reachable.filter((s) => getMcpOAuthMode(s) === "authorization_code");
         commitServers(reachable);
         setOauthChecking(new Set(oauthServers.map((s) => s.server_id)));
         setLoading(false);
@@ -274,7 +289,10 @@ const MCPAppsPanel: React.FC<Props> = ({ accessToken, selectedServers, onChange,
         <span className="text-[11px] text-muted-foreground shrink-0 whitespace-nowrap">{unavailabilityLabel}</span>
       );
     }
-    if (server.auth_type === AUTH_TYPE.OAUTH2) {
+    if (getMcpOAuthMode(server) === "m2m") {
+      return <CheckCircle className="h-3.5 w-3.5 text-success shrink-0" />;
+    }
+    if (getMcpOAuthMode(server) === "authorization_code") {
       if (oauthConnected.has(server.server_id)) {
         return <CheckCircle className="h-3.5 w-3.5 text-success shrink-0" />;
       }
@@ -339,7 +357,10 @@ const MCPAppsPanel: React.FC<Props> = ({ accessToken, selectedServers, onChange,
       if (unavailabilityLabel !== null) {
         return <span className="text-[13px] text-muted-foreground py-2.5 shrink-0">{unavailabilityLabel}</span>;
       }
-      if (detailServer.auth_type !== AUTH_TYPE.OAUTH2) {
+      if (getMcpOAuthMode(detailServer) === "m2m") {
+        return <span className="text-[13px] text-muted-foreground">Authorized</span>;
+      }
+      if (getMcpOAuthMode(detailServer) !== "authorization_code") {
         return (
           <Button
             variant={isConnected ? "outline" : "default"}
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx
index 0e9133d6d3c..d7cf5b6110b 100644
--- a/ui/litellm-dashboard/src/components/networking.tsx
+++ b/ui/litellm-dashboard/src/components/networking.tsx
@@ -4918,6 +4918,24 @@ export const fetchDiscoverableMCPServers = async (accessToken: string) => {
   }
 };
 
+export interface ConnectFlowStatus {
+  state: "unscoped" | "interactive" | "m2m" | "stale";
+  client_origin: string;
+  server_id: string | null;
+  server_name: string | null;
+  connected: boolean | null;
+}
+
+/**
+ * What the gateway says about one in-flight connect flow, read from the sealed HttpOnly
+ * flow cookie rather than from the address bar (LIT-7075): the client asking, the server
+ * the flow is scoped to, and whether that server's vendor OAuth is already done. Sent with
+ * no access token; the flow and session cookies are the credential, exactly as they are for
+ * the finish form this page posts.
+ */
+export const fetchConnectFlow = async (flowHandle: string): Promise<ConnectFlowStatus> =>
+  apiClient.get(`/authorize/flow`, { query: { flow: flowHandle }, credentials: "include" });
+
 export const fetchMCPServers = async (accessToken: string, teamId?: string | null, connectedAppView?: boolean) => {
   try {
     return await apiClient.get(`/v1/mcp/server`, {
diff --git a/ui/litellm-dashboard/src/lib/http/client.test.ts b/ui/litellm-dashboard/src/lib/http/client.test.ts
index c3be1d48b42..0d765371e0c 100644
--- a/ui/litellm-dashboard/src/lib/http/client.test.ts
+++ b/ui/litellm-dashboard/src/lib/http/client.test.ts
@@ -41,6 +41,15 @@ describe("createApiClient", () => {
     expect(init.body).toBeUndefined();
   });
 
+  it("forwards the browser credential policy", async () => {
+    const fetchImpl = vi.fn(async () => okResponse({}));
+    const client = createApiClient({ getBaseUrl: () => "https://proxy.example", fetchImpl });
+
+    await client.get("/authorize/flow", { credentials: "include" });
+
+    expect(fetchImpl.mock.calls[0][1].credentials).toBe("include");
+  });
+
   it("JSON-serializes the body for writes", async () => {
     const fetchImpl = vi.fn(async () => okResponse({}));
     const client = createApiClient({ getBaseUrl: () => "", fetchImpl });
diff --git a/ui/litellm-dashboard/src/lib/http/client.ts b/ui/litellm-dashboard/src/lib/http/client.ts
index b8cfb81211a..c78b2b10011 100644
--- a/ui/litellm-dashboard/src/lib/http/client.ts
+++ b/ui/litellm-dashboard/src/lib/http/client.ts
@@ -25,6 +25,8 @@ export interface RequestOptions {
   query?: QueryParams;
   headers?: Record<string, string>;
   signal?: AbortSignal;
+  /** Send browser cookies with the request; needed for cookie-authenticated proxy routes. */
+  credentials?: RequestCredentials;
 }
 
 export class ApiError extends Error {
@@ -136,7 +138,7 @@ export function createApiClient(config: ApiClientConfig): ApiClient {
   const doFetch: typeof fetch = (input, init) => (fetchImpl ?? fetch)(input, init);
 
   async function request<T = any>(method: HttpMethod, path: string, options: RequestOptions = {}): Promise<T> {
-    const { accessToken, body, rawBody, query, headers: extraHeaders, signal } = options;
+    const { accessToken, body, rawBody, query, headers: extraHeaders, signal, credentials } = options;
 
     const url = appendQuery(`${getBaseUrl()}${path}`, query);
 
@@ -152,7 +154,7 @@ export function createApiClient(config: ApiClientConfig): ApiClient {
       Object.assign(headers, extraHeaders);
     }
 
-    const init: RequestInit = { method, headers, signal };
+    const init: RequestInit = { method, headers, signal, credentials };
     if (rawBody !== undefined) {
       init.body = rawBody;
     } else if (body !== undefined) {
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 6c2311a0ed9..232b366d8ce 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -1169,6 +1169,23 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/authorize/flow": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /** Authorize Flow */
+        get: operations["authorize_flow_authorize_flow_get"];
+        put?: never;
+        post?: never;
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/auto_router/benchmarks": {
         parameters: {
             query?: never;
@@ -41433,6 +41450,37 @@ export interface operations {
             };
         };
     };
+    authorize_flow_authorize_flow_get: {
+        parameters: {
+            query: {
+                flow: string;
+            };
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     get_auto_router_benchmarks_auto_router_benchmarks_get: {
         parameters: {
             query?: {

From 95c0f9db7d92cefafa5e2d9799786b4931c9a390 Mon Sep 17 00:00:00 2001
From: Clement <clement.ho@paradigm.co>
Date: Wed, 9 Sep 2026 01:05:23 +0800
Subject: [PATCH 13/27] fix(router): rank streaming latency routing by raw
 TTFT, not TTFT per token (#40202)

* fix(router): rank streaming latency routing by raw TTFT, not TTFT per token

Latency-based routing divided time-to-first-token by completion_tokens
before storing it, so a deployment that streamed a long answer looked
faster to first token than one that answered briefly. TTFT is now stored
as plain seconds (first token time minus request start) in both the sync
and async success handlers, which is what the routing decision compares.

Non-streaming latency normalization per output token is unchanged.

Claude-Session: https://claude.ai/code/session_01Ttd5Q9ZhRPB4ch5guos3rj

* fix(router): store streaming TTFT under a seconds-only cache key

Workers on the previous release keep writing seconds-per-token samples
under "time_to_first_token" in the shared router cache during a rolling
deploy, so mixing the new raw-seconds samples into the same list averaged
incompatible units. Raw TTFT now lives under "time_to_first_token_seconds"
and routing reads only that key.

Also fix the regression test's token counts: with 50 tokens on the fast
deployment and 500 on the slow one the old per-token formula picks the
slow deployment, so the routing assertion now catches the bug.

Claude-Session: https://claude.ai/code/session_01Ttd5Q9ZhRPB4ch5guos3rj

* test(router): cover the TTFT sliding window from the unit-test shard

Move the TTFT list trimming checks from the CircleCI-only suite into the
mapped unit test file as one sync/async parametrized test, so the changed
lines in lowest_latency.py are exercised by the GitHub unit-test shard
that reports patch coverage.

Claude-Session: https://claude.ai/code/session_01Ttd5Q9ZhRPB4ch5guos3rj
---
 litellm/router_strategy/lowest_latency.py     |  54 +++----
 .../test_lowest_latency_routing.py            | 136 ------------------
 .../router_strategy/test_lowest_latency.py    | 127 ++++++++++++++++
 3 files changed, 149 insertions(+), 168 deletions(-)

diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py
index bb6877a032b..805d4ff9080 100644
--- a/litellm/router_strategy/lowest_latency.py
+++ b/litellm/router_strategy/lowest_latency.py
@@ -32,6 +32,12 @@ def _average_latency(samples: Sequence[float]) -> float:
     return sum(samples) / len(samples)
 
 
+def _ttft_seconds(elapsed: timedelta | float) -> float:
+    if isinstance(elapsed, timedelta):
+        return elapsed.total_seconds()
+    return float(elapsed)
+
+
 class LowestLatencyLoggingHandler(CustomLogger):
     test_flag: bool = False
     logged_success: int = 0
@@ -86,14 +92,13 @@ class LowestLatencyLoggingHandler(CustomLogger):
                     # breaks JSON serialization when the router cache syncs to
                     # Redis (issue #33169)
                     response_ms = response_ms.total_seconds()
-                time_to_first_token_response_time = None
+                time_to_first_token: float | None = None
 
                 if kwargs.get("stream", None) is not None and kwargs["stream"] is True:
                     # only log ttft for streaming request
-                    time_to_first_token_response_time = kwargs.get("completion_start_time", end_time) - start_time
+                    time_to_first_token = _ttft_seconds(kwargs.get("completion_start_time", end_time) - start_time)
 
                 final_value: float = response_ms
-                time_to_first_token: float | None = None
                 total_tokens = 0
 
                 if isinstance(response_obj, ModelResponse):
@@ -111,13 +116,6 @@ class LowestLatencyLoggingHandler(CustomLogger):
                         else:
                             final_value = response_seconds
 
-                        if time_to_first_token_response_time is not None:
-                            if isinstance(time_to_first_token_response_time, timedelta):
-                                ttft_seconds = time_to_first_token_response_time.total_seconds()
-                            else:
-                                ttft_seconds = time_to_first_token_response_time
-                            time_to_first_token = safe_divide_seconds(ttft_seconds, completion_tokens)
-
                 # ------------
                 # Update usage
                 # ------------
@@ -138,14 +136,14 @@ class LowestLatencyLoggingHandler(CustomLogger):
                 ## Time to first token
                 if time_to_first_token is not None:
                     if (
-                        len(request_count_dict[id].get("time_to_first_token", []))
+                        len(request_count_dict[id].get("time_to_first_token_seconds", []))
                         < self.routing_args.max_latency_list_size
                     ):
-                        request_count_dict[id].setdefault("time_to_first_token", []).append(time_to_first_token)
+                        request_count_dict[id].setdefault("time_to_first_token_seconds", []).append(time_to_first_token)
                     else:
-                        request_count_dict[id]["time_to_first_token"] = request_count_dict[id]["time_to_first_token"][
-                            1:
-                        ] + [time_to_first_token]
+                        request_count_dict[id]["time_to_first_token_seconds"] = request_count_dict[id][
+                            "time_to_first_token_seconds"
+                        ][1:] + [time_to_first_token]
 
                 if precise_minute not in request_count_dict[id]:
                     request_count_dict[id][precise_minute] = {}
@@ -252,7 +250,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
                     {model_group}_map: {
                         id: {
                             "latency": [..]
-                            "time_to_first_token": [..]
+                            "time_to_first_token_seconds": [..]
                             f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
                         }
                     }
@@ -273,14 +271,13 @@ class LowestLatencyLoggingHandler(CustomLogger):
                     # breaks JSON serialization when the router cache syncs to
                     # Redis (issue #33169)
                     response_ms = response_ms.total_seconds()
-                time_to_first_token_response_time = None
+                time_to_first_token: float | None = None
                 if kwargs.get("stream", None) is not None and kwargs["stream"] is True:
                     # only log ttft for streaming request
-                    time_to_first_token_response_time = kwargs.get("completion_start_time", end_time) - start_time
+                    time_to_first_token = _ttft_seconds(kwargs.get("completion_start_time", end_time) - start_time)
 
                 final_value: float = response_ms
                 total_tokens = 0
-                time_to_first_token: float | None = None
 
                 if isinstance(response_obj, ModelResponse):
                     _usage: Final = getattr(response_obj, "usage", None)
@@ -296,13 +293,6 @@ class LowestLatencyLoggingHandler(CustomLogger):
                             final_value = float(normalized_value)
                         else:
                             final_value = response_seconds
-
-                        if time_to_first_token_response_time is not None:
-                            if isinstance(time_to_first_token_response_time, timedelta):
-                                ttft_seconds = time_to_first_token_response_time.total_seconds()
-                            else:
-                                ttft_seconds = time_to_first_token_response_time
-                            time_to_first_token = safe_divide_seconds(ttft_seconds, completion_tokens)
                 # ------------
                 # Update usage
                 # ------------
@@ -328,14 +318,14 @@ class LowestLatencyLoggingHandler(CustomLogger):
                 ## Time to first token
                 if time_to_first_token is not None:
                     if (
-                        len(request_count_dict[id].get("time_to_first_token", []))
+                        len(request_count_dict[id].get("time_to_first_token_seconds", []))
                         < self.routing_args.max_latency_list_size
                     ):
-                        request_count_dict[id].setdefault("time_to_first_token", []).append(time_to_first_token)
+                        request_count_dict[id].setdefault("time_to_first_token_seconds", []).append(time_to_first_token)
                     else:
-                        request_count_dict[id]["time_to_first_token"] = request_count_dict[id]["time_to_first_token"][
-                            1:
-                        ] + [time_to_first_token]
+                        request_count_dict[id]["time_to_first_token_seconds"] = request_count_dict[id][
+                            "time_to_first_token_seconds"
+                        ][1:] + [time_to_first_token]
 
                 if precise_minute not in request_count_dict[id]:
                     request_count_dict[id][precise_minute] = {}
@@ -433,7 +423,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
                 or float("inf")
             )
             item_latency = item_map.get("latency", [])
-            item_ttft_latency = item_map.get("time_to_first_token", [])
+            item_ttft_latency = item_map.get("time_to_first_token_seconds", [])
             item_rpm = item_map.get(precise_minute, {}).get("rpm", 0)
             item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)
 
diff --git a/tests/local_testing/test_lowest_latency_routing.py b/tests/local_testing/test_lowest_latency_routing.py
index 598b1dbcaf9..aba4500199a 100644
--- a/tests/local_testing/test_lowest_latency_routing.py
+++ b/tests/local_testing/test_lowest_latency_routing.py
@@ -1077,73 +1077,6 @@ async def test_latency_list_trimming_discards_oldest_entry_async():
         ), f"Oldest latency {oldest_latency} should have been discarded"
 
 
-def test_ttft_list_trimming_discards_oldest_entry():
-    """
-    The time_to_first_token list trims the oldest entry when full, matching
-    the behavior of the latency list.
-    """
-    max_size = 3
-    test_cache = DualCache()
-    lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, routing_args={"max_latency_list_size": max_size}
-    )
-
-    model_group = "gpt-3.5-turbo"
-    deployment_id = "test-deployment"
-
-    ttft_values = []
-    for i in range(max_size + 1):
-        start_time = time.time()
-        expected_ttft = float(i + 1) * 0.1  # 0.1, 0.2, 0.3, 0.4
-        completion_start_time = start_time + expected_ttft
-        end_time = start_time + float(i + 1)
-        ttft_values.append(expected_ttft)
-
-        kwargs = {
-            "litellm_params": {
-                "metadata": {
-                    "model_group": model_group,
-                    "deployment": "azure/gpt-4.1-mini",
-                },
-                "model_info": {"id": deployment_id},
-            },
-            "stream": True,
-            "completion_start_time": completion_start_time,
-        }
-        # TTFT is only recorded when response_obj is a ModelResponse.
-        response_obj = litellm.ModelResponse(
-            usage=litellm.Usage(completion_tokens=1, total_tokens=1)
-        )
-
-        lowest_latency_logger.log_success_event(
-            response_obj=response_obj,
-            kwargs=kwargs,
-            start_time=start_time,
-            end_time=end_time,
-        )
-
-    latency_key = f"{model_group}_map"
-    cached_data = test_cache.get_cache(key=latency_key)
-    ttft_list = cached_data[deployment_id].get("time_to_first_token", [])
-
-    assert (
-        len(ttft_list) == max_size
-    ), f"Expected {max_size} entries, got {len(ttft_list)}"
-
-    newest_ttft = ttft_values[-1]
-    oldest_ttft = ttft_values[0]
-    tolerance = 0.05
-
-    assert (
-        abs(ttft_list[-1] - newest_ttft) < tolerance
-    ), f"Newest TTFT {newest_ttft} should be at end of list"
-
-    for ttft in ttft_list:
-        assert (
-            abs(ttft - oldest_ttft) > tolerance
-        ), f"Oldest TTFT {oldest_ttft} should have been discarded"
-
-
 @pytest.mark.asyncio
 async def test_timeout_penalty_discards_oldest_entry():
     """
@@ -1269,72 +1202,3 @@ def test_list_order_preserved_after_multiple_trims():
         assert (
             abs(latency_list[i] - expected) < tolerance
         ), f"At index {i}, expected ~{expected}, got {latency_list[i]}"
-
-
-@pytest.mark.asyncio
-async def test_ttft_list_trimming_discards_oldest_entry_async():
-    """
-    Async counterpart: the time_to_first_token list trims the oldest entry
-    when full. Exercises the async_log_success_event TTFT path, which only
-    runs when response_obj is a ModelResponse and the call is marked as
-    streaming with a completion_start_time.
-    """
-    max_size = 3
-    test_cache = DualCache()
-    lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, routing_args={"max_latency_list_size": max_size}
-    )
-
-    model_group = "gpt-3.5-turbo"
-    deployment_id = "test-deployment"
-
-    ttft_values = []
-    for i in range(max_size + 1):
-        start_time = time.time()
-        expected_ttft = float(i + 1) * 0.1  # 0.1, 0.2, 0.3, 0.4
-        completion_start_time = start_time + expected_ttft
-        end_time = start_time + float(i + 1)
-        ttft_values.append(expected_ttft)
-
-        kwargs = {
-            "litellm_params": {
-                "metadata": {
-                    "model_group": model_group,
-                    "deployment": "azure/gpt-4.1-mini",
-                },
-                "model_info": {"id": deployment_id},
-            },
-            "stream": True,
-            "completion_start_time": completion_start_time,
-        }
-        response_obj = litellm.ModelResponse(
-            usage=litellm.Usage(completion_tokens=1, total_tokens=1)
-        )
-
-        await lowest_latency_logger.async_log_success_event(
-            response_obj=response_obj,
-            kwargs=kwargs,
-            start_time=start_time,
-            end_time=end_time,
-        )
-
-    latency_key = f"{model_group}_map"
-    cached_data = await test_cache.async_get_cache(key=latency_key)
-    ttft_list = cached_data[deployment_id].get("time_to_first_token", [])
-
-    assert (
-        len(ttft_list) == max_size
-    ), f"Expected {max_size} entries, got {len(ttft_list)}"
-
-    newest_ttft = ttft_values[-1]
-    oldest_ttft = ttft_values[0]
-    tolerance = 0.05
-
-    assert (
-        abs(ttft_list[-1] - newest_ttft) < tolerance
-    ), f"Newest TTFT {newest_ttft} should be at end of list"
-
-    for ttft in ttft_list:
-        assert (
-            abs(ttft - oldest_ttft) > tolerance
-        ), f"Oldest TTFT {oldest_ttft} should have been discarded"
diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py
index eb02459be68..812d7bbff32 100644
--- a/tests/test_litellm/router_strategy/test_lowest_latency.py
+++ b/tests/test_litellm/router_strategy/test_lowest_latency.py
@@ -165,6 +165,133 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds():
     json.dumps({"latency": latencies})
 
 
+MODEL_GROUP = "gpt-4o-mini"
+FAST_TTFT_ID = "fast-ttft-short-output"
+SLOW_TTFT_ID = "slow-ttft-long-output"
+STREAMING_DEPLOYMENTS = [
+    {"model_info": {"id": FAST_TTFT_ID}, "litellm_params": {}},
+    {"model_info": {"id": SLOW_TTFT_ID}, "litellm_params": {}},
+]
+
+
+def _streaming_kwargs(deployment_id: str, start_time: datetime, ttft_seconds: float):
+    return {
+        "litellm_params": {
+            "metadata": {"model_group": MODEL_GROUP},
+            "model_info": {"id": deployment_id},
+        },
+        "stream": True,
+        "completion_start_time": start_time + timedelta(seconds=ttft_seconds),
+    }
+
+
+def _recorded_ttft(cache: DualCache, deployment_id: str):
+    cached = cache.get_cache(key=f"{MODEL_GROUP}_map") or {}
+    return cached.get(deployment_id, {}).get("time_to_first_token_seconds", [])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
+async def test_streaming_ttft_ranking_ignores_completion_length(sync_mode: bool):
+    """Deployment A: TTFT 1s, 50 completion tokens. Deployment B: TTFT 3s, 500
+    completion tokens. Dividing TTFT by completion tokens made B look faster
+    (3/500 = 0.006 beats 1/50 = 0.02); actual TTFT must win."""
+    cache = DualCache()
+    handler = LowestLatencyLoggingHandler(router_cache=cache)
+    start_time = datetime(2026, 1, 1, 12, 0, 0)
+    end_time = start_time + timedelta(seconds=10)
+
+    samples = (
+        (FAST_TTFT_ID, 1.0, 50),
+        (SLOW_TTFT_ID, 3.0, 500),
+    )
+    for deployment_id, ttft, completion_tokens in samples:
+        kwargs = _streaming_kwargs(deployment_id, start_time, ttft)
+        response_obj = _chat_response(completion_tokens=completion_tokens)
+        if sync_mode:
+            handler.log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=end_time
+            )
+        else:
+            await handler.async_log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=end_time
+            )
+
+    assert _recorded_ttft(cache, FAST_TTFT_ID) == [pytest.approx(1.0)]
+    assert _recorded_ttft(cache, SLOW_TTFT_ID) == [pytest.approx(3.0)]
+
+    request_kwargs = {"stream": True, "metadata": {}}
+    if sync_mode:
+        picked = handler.get_available_deployments(
+            model_group=MODEL_GROUP, healthy_deployments=STREAMING_DEPLOYMENTS, request_kwargs=request_kwargs
+        )
+    else:
+        picked = await handler.async_get_available_deployments(
+            model_group=MODEL_GROUP, healthy_deployments=STREAMING_DEPLOYMENTS, request_kwargs=request_kwargs
+        )
+
+    assert picked is not None
+    assert picked["model_info"]["id"] == FAST_TTFT_ID
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
+async def test_ttft_window_keeps_newest_samples_when_full(sync_mode: bool):
+    """Float timestamps, as the SDK passes them. Once max_latency_list_size
+    samples exist the oldest TTFT is dropped so the window slides."""
+    max_size = 3
+    cache = DualCache()
+    handler = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"max_latency_list_size": max_size})
+    start_time = 1_700_000_000.0
+    ttfts = (0.1, 0.2, 0.3, 0.4)
+
+    for ttft in ttfts:
+        kwargs = {
+            "litellm_params": {
+                "metadata": {"model_group": MODEL_GROUP},
+                "model_info": {"id": FAST_TTFT_ID},
+            },
+            "stream": True,
+            "completion_start_time": start_time + ttft,
+        }
+        response_obj = _chat_response(completion_tokens=1)
+        if sync_mode:
+            handler.log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=start_time + 1.0
+            )
+        else:
+            await handler.async_log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=start_time + 1.0
+            )
+
+    assert _recorded_ttft(cache, FAST_TTFT_ID) == [pytest.approx(ttft) for ttft in ttfts[-max_size:]]
+
+
+@pytest.mark.asyncio
+async def test_streaming_routing_ignores_per_token_ttft_samples_from_older_workers():
+    """Workers on the previous release share the Redis map and keep writing
+    seconds-per-token under the old "time_to_first_token" key during a rolling
+    deploy. Those samples favor SLOW; routing must only read the seconds key."""
+    cache = DualCache()
+    handler = LowestLatencyLoggingHandler(router_cache=cache)
+    cache.set_cache(
+        key=f"{MODEL_GROUP}_map",
+        value={
+            FAST_TTFT_ID: {"time_to_first_token": [0.02], "time_to_first_token_seconds": [1.0]},
+            SLOW_TTFT_ID: {"time_to_first_token": [0.006], "time_to_first_token_seconds": [3.0]},
+        },
+    )
+
+    picked = await handler.async_get_available_deployments(
+        model_group=MODEL_GROUP,
+        healthy_deployments=STREAMING_DEPLOYMENTS,
+        request_kwargs={"stream": True, "metadata": {}},
+    )
+
+    assert picked is not None
+    assert picked["model_info"]["id"] == FAST_TTFT_ID
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "cached_entry",

From 978f93ce9cf3e2748494a1c8b67b48488930b06b Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:05:32 -0700
Subject: [PATCH 14/27] fix(tool_permission): log expected skip and deny events
 below WARNING (#40208)

The no-tools skip path in async_pre_call_hook now logs at DEBUG and the
denied-by-rule messages in async_pre_call_hook and _evaluate_tool_calls
log at INFO. The malformed tool arguments warning is unchanged. Adds
regression tests pinning each level

Co-authored-by: yucheng <yucheng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../guardrail_hooks/tool_permission.py        |  6 +-
 .../guardrail_hooks/test_tool_permission.py   | 75 +++++++++++++++++++
 2 files changed, 78 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
index a8b33109900..e20f0b320b9 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
@@ -547,7 +547,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
 
         for _tool_call, is_allowed, _rule_id, message in checked:
             if not is_allowed and message is not None:
-                verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message)
+                verbose_proxy_logger.info("Tool Permission Guardrail: %s", message)
                 if self.on_disallowed_action == "block":
                     raise GuardrailRaisedException(
                         guardrail_name=self.guardrail_name, message=message, blocked_content=True
@@ -809,7 +809,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
 
         new_tools: Final = self._collect_request_tools(data)
         if not new_tools:
-            verbose_proxy_logger.warning(
+            verbose_proxy_logger.debug(
                 "Tool Permission Guardrail: not running guardrail. No tools or functions in data"
             )
             return data
@@ -820,7 +820,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
             is_allowed, _, message = self._check_tool_permission(tool_name, tool_type)
 
             if not is_allowed and message is not None:
-                verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message)
+                verbose_proxy_logger.info("Tool Permission Guardrail: %s", message)
                 if self.on_disallowed_action == "block":
                     raise HTTPException(
                         status_code=400,
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py
index 0dbd4591ac9..87a1b84acc5 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py
@@ -3,6 +3,7 @@ Unit tests for Tool Permission Guardrail (OpenAI tool_calls semantics)
 """
 
 import json
+import logging
 import re
 from unittest.mock import patch
 
@@ -520,6 +521,80 @@ class TestToolPermissionGuardrail:
                 )
         assert excinfo.value.status_code == 400
 
+    @pytest.mark.asyncio
+    async def test_async_pre_call_hook_without_tools_logs_skip_at_debug(self, caplog):
+        data = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
+                result = await self.guardrail.async_pre_call_hook(
+                    user_api_key_dict=UserAPIKeyAuth(),
+                    cache=DualCache(default_in_memory_ttl=1),
+                    data=data,
+                    call_type="completion",
+                )
+
+        assert result is data
+        skip_levels = [r.levelno for r in caplog.records if "No tools or functions in data" in r.getMessage()]
+        assert skip_levels == [logging.DEBUG]
+        assert [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] == []
+
+    @pytest.mark.asyncio
+    async def test_async_pre_call_hook_denied_tool_logs_at_info(self, caplog):
+        data = {"tools": [{"type": "function", "function": {"name": "Read"}}]}
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
+                with pytest.raises(HTTPException):
+                    await self.guardrail.async_pre_call_hook(
+                        user_api_key_dict=UserAPIKeyAuth(),
+                        cache=DualCache(default_in_memory_ttl=1),
+                        data=data,
+                        call_type="completion",
+                    )
+
+        denied_levels = [
+            r.levelno
+            for r in caplog.records
+            if r.getMessage() == "Tool Permission Guardrail: Tool 'Read' denied by rule 'deny_read'"
+        ]
+        assert denied_levels == [logging.INFO]
+        assert [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] == []
+
+    @pytest.mark.asyncio
+    async def test_async_post_call_success_hook_denied_tool_logs_at_info(self, caplog):
+        tool_call = {"function": {"name": "Read", "arguments": "{}"}, "type": "function"}
+        response = ModelResponse(choices=[Choices(message={"tool_calls": [tool_call]})])
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
+                with pytest.raises(GuardrailRaisedException):
+                    await self.guardrail.async_post_call_success_hook(
+                        data={"guardrails": ["test-tool-permission"]},
+                        user_api_key_dict=UserAPIKeyAuth(),
+                        response=response,
+                    )
+
+        denied_levels = [
+            r.levelno
+            for r in caplog.records
+            if r.getMessage() == "Tool Permission Guardrail: Tool 'Read' denied by rule 'deny_read'"
+        ]
+        assert denied_levels == [logging.INFO]
+        assert [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] == []
+
+    def test_parse_tool_call_arguments_malformed_json_logs_warning(self, caplog):
+        tool_call = ChatCompletionMessageToolCall(function={"name": "Bash", "arguments": "{not json"}, id="call_1")
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            parsed, error = self.guardrail._parse_tool_call_arguments(tool_call)
+
+        assert parsed is None
+        assert error == "arguments could not be parsed"
+        warning_messages = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
+        assert len(warning_messages) == 1
+        assert warning_messages[0].startswith("Tool Permission Guardrail: Failed to decode arguments for tool Bash")
+
     @pytest.mark.asyncio
     async def test_async_pre_call_hook_blocks_legacy_functions(self):
         data = {

From 99824533ff7a99c07e0c768f61fcd5b5204a7794 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:06:53 -0700
Subject: [PATCH 15/27] fix(proxy): kill the whole prisma process group when a
 boot migration command times out (#39509)

* fix(proxy): kill the whole prisma process group when a boot migration command times out

* fix(proxy): keep boot alive when the installed proxy extras has no prisma runner
---
 litellm/proxy/db/check_migration.py           | 39 ++++++---
 litellm/proxy/db/prisma_client.py             | 22 ++++--
 tests/test_litellm/proxy/db/conftest.py       | 79 +++++++++++++++++++
 .../proxy/db/test_check_migration.py          | 32 ++++++++
 .../proxy/db/test_prisma_client.py            | 59 +++++++++-----
 5 files changed, 197 insertions(+), 34 deletions(-)

diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py
index b7a07d4eeea..6e2a06c96e1 100644
--- a/litellm/proxy/db/check_migration.py
+++ b/litellm/proxy/db/check_migration.py
@@ -46,17 +46,32 @@ def extract_sql_commands(diff_output: str) -> list[str]:
 
 def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
     """Checks for differences between current database and Prisma schema.
+
+    Never raises: a diff that cannot be produced, because the runner is missing,
+    because the command failed, or because it outlived its budget, is reported as
+    "no diff" so boot continues.
+
     Returns:
         A tuple containing:
         - A boolean indicating if differences were found (True) or not (False).
-        - A string with the diff output or error message.
-    Raises:
-        subprocess.CalledProcessError: If the Prisma command fails.
-        Exception: For any other errors during execution.
+        - The SQL commands that would close the diff, empty when there is none.
     """
-    verbose_logger.debug("Checking for Prisma schema diff...")
     try:
-        result: Final = subprocess.run(
+        from litellm_proxy_extras.prisma_toolchain import (
+            PRISMA_COMMAND_TIMEOUT_ENV_VAR,
+            prisma_command_timeout,
+            run_prisma,
+        )
+    except ImportError as e:
+        print(  # noqa: T201  # boot-time operator output, same channel as this helper's other messages
+            f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}"
+        )
+        return False, []
+
+    verbose_logger.debug("Checking for Prisma schema diff...")
+    timeout: Final = prisma_command_timeout()
+    try:
+        result: Final = run_prisma(
             [
                 "prisma",
                 "migrate",
@@ -67,12 +82,10 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
                 "./schema.prisma",
                 "--script",
             ],
-            capture_output=True,
-            text=True,
-            check=True,
+            timeout=timeout,
+            env=os.environ.copy(),
         )
 
-        # return True, "Migration diff generated successfully."
         sql_commands: Final = extract_sql_commands(result.stdout)
 
         if sql_commands:
@@ -83,6 +96,12 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
             return True, sql_commands
         else:
             return False, []
+    except subprocess.TimeoutExpired:
+        print(  # noqa: T201  # boot-time operator output, same channel as this helper's other messages
+            f"Timed out after {timeout}s generating the migration diff. "
+            f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer."
+        )
+        return False, []
     except subprocess.CalledProcessError as e:
         error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}"
         print(error_message)  # noqa: T201
diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py
index 2190ae55fd2..9ea2432f2f5 100644
--- a/litellm/proxy/db/prisma_client.py
+++ b/litellm/proxy/db/prisma_client.py
@@ -937,9 +937,17 @@ class PrismaManager:
                         use_v2_resolver=use_v2_resolver,
                     )
                 else:
+                    try:
+                        from litellm_proxy_extras.prisma_toolchain import (
+                            prisma_command_timeout,
+                            run_prisma,
+                        )
+                    except ImportError as e:
+                        verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e)
+                        return False
+
                     PrismaManager._raise_if_partitioned_spend_logs()
-                    # Use prisma db push with increased timeout
-                    subprocess.run(
+                    run_prisma(
                         [
                             "prisma",
                             "db",
@@ -947,13 +955,15 @@ class PrismaManager:
                             "--accept-data-loss",
                             "--skip-generate",
                         ],
-                        timeout=60,
-                        check=True,
+                        timeout=prisma_command_timeout(),
+                        env=os.environ.copy(),
+                        stdout=None,
+                        stderr=None,
                     )
                     PrismaManager._apply_replica_identity_full_if_requested()
                     return True
-            except subprocess.TimeoutExpired:
-                verbose_proxy_logger.warning("Attempt %s timed out", attempt + 1)
+            except subprocess.TimeoutExpired as e:
+                verbose_proxy_logger.warning("Attempt %s timed out after %.0fs", attempt + 1, e.timeout)
                 time.sleep(random.randrange(5, 15))
             except subprocess.CalledProcessError as e:
                 attempts_left = 3 - attempt
diff --git a/tests/test_litellm/proxy/db/conftest.py b/tests/test_litellm/proxy/db/conftest.py
index 6f67b91ac1d..03d7ea81257 100644
--- a/tests/test_litellm/proxy/db/conftest.py
+++ b/tests/test_litellm/proxy/db/conftest.py
@@ -1,5 +1,11 @@
+import json
 import os
+import signal
+import sys
+import time
 from collections.abc import Generator
+from dataclasses import dataclass
+from pathlib import Path
 from typing import Optional
 
 import pytest
@@ -75,3 +81,76 @@ def reset_entra_token_provider_cache() -> Generator[None, None, None]:
 def unset_database_url(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("DATABASE_URL", "about-to-be-unset")
     monkeypatch.delenv("DATABASE_URL")
+
+
+FAKE_PRISMA_CLI = """#!{python}
+import json
+import os
+import pathlib
+import subprocess
+import sys
+import time
+
+calls_file = pathlib.Path(os.environ["FAKE_PRISMA_CALLS"])
+earlier_calls = calls_file.read_text().splitlines() if calls_file.exists() else []
+with calls_file.open("a") as log:
+    print(json.dumps(sys.argv[1:]), file=log)
+if not earlier_calls and os.environ.get("FAKE_PRISMA_HANG_FIRST"):
+    grandchild = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"])
+    pathlib.Path(os.environ["FAKE_PRISMA_GRANDCHILD_PIDFILE"]).write_text(str(grandchild.pid))
+    time.sleep(600)
+sys.exit(0)
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class FakePrismaCli:
+    """A stand-in `prisma` on PATH, recording every invocation.
+
+    With FAKE_PRISMA_HANG_FIRST set it hangs on its first call from a process tree
+    of its own, the way the real CLI wraps Node around a Rust schema engine, so a
+    timeout that kills only the direct child leaves the rest of that tree running.
+    """
+
+    calls_file: Path
+    grandchild_pidfile: Path
+
+    @property
+    def calls(self) -> list[list[str]]:
+        if not self.calls_file.exists():
+            return []
+        return [json.loads(line) for line in self.calls_file.read_text().splitlines()]
+
+    def grandchild_is_gone(self, within_seconds: float) -> bool:
+        deadline = time.monotonic() + within_seconds
+        while time.monotonic() < deadline:
+            try:
+                os.kill(int(self.grandchild_pidfile.read_text()), 0)
+            except ProcessLookupError:
+                return True
+            time.sleep(0.05)
+        return False
+
+
+@pytest.fixture
+def fake_prisma_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[FakePrismaCli, None, None]:
+    bin_dir = tmp_path / "fakebin"
+    bin_dir.mkdir()
+    script = bin_dir / "prisma"
+    script.write_text(FAKE_PRISMA_CLI.format(python=sys.executable))
+    script.chmod(0o755)
+    cli = FakePrismaCli(
+        calls_file=tmp_path / "calls.jsonl",
+        grandchild_pidfile=tmp_path / "grandchild.pid",
+    )
+    monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}")
+    monkeypatch.setenv("FAKE_PRISMA_CALLS", str(cli.calls_file))
+    monkeypatch.setenv("FAKE_PRISMA_GRANDCHILD_PIDFILE", str(cli.grandchild_pidfile))
+    monkeypatch.setenv("LITELLM_PRISMA_COMMAND_TIMEOUT", "1")
+    monkeypatch.delenv("FAKE_PRISMA_HANG_FIRST", raising=False)
+    yield cli
+    if cli.grandchild_pidfile.exists():
+        try:
+            os.kill(int(cli.grandchild_pidfile.read_text()), signal.SIGKILL)
+        except ProcessLookupError:
+            pass
diff --git a/tests/test_litellm/proxy/db/test_check_migration.py b/tests/test_litellm/proxy/db/test_check_migration.py
index 9e2f6a1089c..74a6841cb23 100644
--- a/tests/test_litellm/proxy/db/test_check_migration.py
+++ b/tests/test_litellm/proxy/db/test_check_migration.py
@@ -37,3 +37,35 @@ def test_check_migration_out_of_sync(mocker):
     check_migration.verbose_logger.exception.assert_called_once()
     actual_message = check_migration.verbose_logger.exception.call_args[0][0]
     assert "prisma schema out of sync with db" in actual_message
+
+
+@pytest.mark.timeout(30)
+def test_migrate_diff_stops_at_its_budget_and_takes_its_process_tree_with_it(fake_prisma_cli, monkeypatch):
+    """
+    `prisma migrate diff` ran unbounded, so a database that never answers hung boot
+    before uvicorn ever started, and interrupting the proxy orphaned the schema engine.
+    """
+    from litellm.proxy.db.check_migration import check_prisma_schema_diff_helper
+
+    monkeypatch.setenv("FAKE_PRISMA_HANG_FIRST", "1")
+
+    assert check_prisma_schema_diff_helper("postgresql://u:p@localhost:9/x") == (False, [])
+    assert fake_prisma_cli.calls == [
+        ["migrate", "diff", "--from-url", "postgresql://u:p@localhost:9/x",
+         "--to-schema-datamodel", "./schema.prisma", "--script"]
+    ]
+    assert fake_prisma_cli.grandchild_is_gone(within_seconds=5)
+
+
+def test_migrate_diff_without_the_prisma_runner_skips_instead_of_crashing_boot(monkeypatch):
+    """
+    Boot calls this helper directly, so an ImportError here takes the proxy down before
+    uvicorn starts. An install without the runner must lose the diagnostic, not the proxy.
+    """
+    import sys
+
+    from litellm.proxy.db.check_migration import check_prisma_schema_diff_helper
+
+    monkeypatch.setitem(sys.modules, "litellm_proxy_extras.prisma_toolchain", None)
+
+    assert check_prisma_schema_diff_helper("postgresql://u:p@localhost:9/x") == (False, [])
diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py
index f0983d6bf62..7f43e483557 100644
--- a/tests/test_litellm/proxy/db/test_prisma_client.py
+++ b/tests/test_litellm/proxy/db/test_prisma_client.py
@@ -10,7 +10,7 @@ from fastapi.testclient import TestClient
 
 
 
-from litellm.proxy.db.prisma_client import PrismaWrapper, should_update_prisma_schema
+from litellm.proxy.db.prisma_client import PrismaManager, PrismaWrapper, should_update_prisma_schema
 
 
 @pytest.fixture(autouse=True)
@@ -193,7 +193,10 @@ async def test_recreate_prisma_client_recovers_from_disconnected_client(
     mock_new_prisma.connect.assert_awaited_once()
 
 
-def test_db_push_applies_replica_identity_full_when_requested(monkeypatch):
+DB_PUSH_ARGV = ["db", "push", "--accept-data-loss", "--skip-generate"]
+
+
+def test_db_push_applies_replica_identity_full_when_requested(monkeypatch, fake_prisma_cli, unset_database_url):
     """`prisma db push` bypasses litellm-proxy-extras, so it needs its own call
     into the opt-in REPLICA IDENTITY FULL step."""
     from litellm.proxy.db.prisma_client import PrismaManager
@@ -208,14 +211,13 @@ def test_db_push_applies_replica_identity_full_when_requested(monkeypatch):
         staticmethod(lambda: applied.append(True)),
     )
 
-    with patch("litellm.proxy.db.prisma_client.subprocess.run") as mock_run:
-        assert PrismaManager.setup_database(use_migrate=False) is True
+    assert PrismaManager.setup_database(use_migrate=False) is True
 
-    assert mock_run.call_args[0][0][:3] == ["prisma", "db", "push"]
+    assert fake_prisma_cli.calls == [DB_PUSH_ARGV]
     assert applied == [True]
 
 
-def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch):
+def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch, fake_prisma_cli, unset_database_url):
     """A doc-partitioned LiteLLM_SpendLogs makes `prisma db push` rewrite the
     primary key back to ("request_id"), which Postgres rejects; the guard must
     fail fast with guidance instead of running the push."""
@@ -228,29 +230,23 @@ def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch):
     monkeypatch.setattr(
         ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
     )
-    with patch(  # test-quality-ok: subprocess.run is the external prisma CLI boundary, asserted never reached
-        "litellm.proxy.db.prisma_client.subprocess.run"
-    ) as mock_run:
-        with pytest.raises(RuntimeError) as err:
-            PrismaManager.setup_database(use_migrate=False)
+    with pytest.raises(RuntimeError) as err:
+        PrismaManager.setup_database(use_migrate=False)
 
     assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR
-    mock_run.assert_not_called()
+    assert fake_prisma_cli.calls == []
 
 
-def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch):
+def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch, fake_prisma_cli, unset_database_url):
     from litellm.proxy.db.prisma_client import PrismaManager
     from litellm_proxy_extras.utils import ProxyExtrasDBManager
 
     monkeypatch.setattr(
         ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: False)
     )
-    with patch(  # test-quality-ok: subprocess.run is the external prisma CLI boundary, not SDK logic
-        "litellm.proxy.db.prisma_client.subprocess.run"
-    ) as mock_run:
-        assert PrismaManager.setup_database(use_migrate=False) is True
+    assert PrismaManager.setup_database(use_migrate=False) is True
 
-    assert mock_run.call_args[0][0][:3] == ["prisma", "db", "push"]
+    assert fake_prisma_cli.calls == [DB_PUSH_ARGV]
 
 
 def _entra_jwt(expires_in_seconds: int) -> str:
@@ -377,3 +373,30 @@ def test_minting_without_the_database_env_vars_names_them(azure_env, monkeypatch
 
     with pytest.raises(RuntimeError, match="DATABASE_HOST"):
         wrapper.get_rds_iam_token()
+
+
+@pytest.mark.timeout(45)
+def test_db_push_timeout_takes_its_process_tree_with_it(fake_prisma_cli, unset_database_url, monkeypatch):
+    """
+    A timed-out `db push` used to leave Node and the schema engine writing the schema,
+    so the next attempt pushed into a database the abandoned one was still mutating.
+    """
+    monkeypatch.delenv("LITELLM_SET_REPLICA_IDENTITY_FULL", raising=False)
+    monkeypatch.setenv("FAKE_PRISMA_HANG_FIRST", "1")
+
+    assert PrismaManager.setup_database(use_migrate=False) is True
+    assert fake_prisma_cli.calls == [DB_PUSH_ARGV, DB_PUSH_ARGV]
+    assert fake_prisma_cli.grandchild_is_gone(within_seconds=5)
+
+
+def test_db_push_without_the_prisma_runner_fails_the_migration_instead_of_crashing_boot(
+    fake_prisma_cli, unset_database_url, monkeypatch
+):
+    """
+    An ImportError out of setup_database escapes the caller's RuntimeError handler and
+    kills boot, bypassing the operator's enforce_prisma_migration_check choice.
+    """
+    monkeypatch.setitem(sys.modules, "litellm_proxy_extras.prisma_toolchain", None)
+
+    assert PrismaManager.setup_database(use_migrate=False) is False
+    assert fake_prisma_cli.calls == []

From a85c3152ca59b5b56d1168e2a384e29a4fb0b861 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:08:24 -0700
Subject: [PATCH 16/27] fix(proxy): load db credentials in the model reconcile
 so a worker never serves a model before its credential (#39876)

* fix(proxy): load db credentials inside the model reconcile so a worker never serves a model before its credential

* fix(proxy): load db credentials in the model read-through so a request miss never adds a model before its credential

* fix(proxy): read credentials from the writer db before the router update and look a credential up once

* test(proxy): assert the credential is loaded when db models reach the router instead of the call order
---
 .../litellm_core_utils/credential_accessor.py |  15 ++-
 .../common_utils/registry_read_through.py     |   1 +
 litellm/proxy/proxy_server.py                 |  31 +----
 litellm/utils.py                              |  18 ++-
 .../common_utils/test_config_sync_pubsub.py   |   7 +-
 .../test_registry_read_through.py             |  45 +++++++
 .../test_scheduled_job_stagger.py             |   2 +-
 .../proxy/proxy_server/test_proxy_config.py   | 123 ++++++++++++++++++
 tests/test_litellm/proxy/test_proxy_server.py |  68 +++++-----
 tests/test_litellm/test_utils.py              |  44 +++++++
 10 files changed, 278 insertions(+), 76 deletions(-)

diff --git a/litellm/litellm_core_utils/credential_accessor.py b/litellm/litellm_core_utils/credential_accessor.py
index 7071750b970..7b4e8240b69 100644
--- a/litellm/litellm_core_utils/credential_accessor.py
+++ b/litellm/litellm_core_utils/credential_accessor.py
@@ -7,16 +7,19 @@ from litellm.types.utils import CredentialItem
 
 
 class CredentialAccessor:
+    @staticmethod
+    def find_credential(credential_name: str) -> CredentialItem | None:
+        return next(
+            (credential for credential in litellm.credential_list if credential.credential_name == credential_name),
+            None,
+        )
+
     @staticmethod
     def get_credential_values(credential_name: str) -> dict:
         """Safe accessor for credentials."""
 
-        if not litellm.credential_list:
-            return {}
-        for credential in litellm.credential_list:
-            if credential.credential_name == credential_name:
-                return credential.credential_values.copy()
-        return {}
+        credential: Final = CredentialAccessor.find_credential(credential_name)
+        return {} if credential is None else credential.credential_values.copy()
 
     @staticmethod
     def upsert_credentials(credentials: list[CredentialItem]):
diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py
index 460b348e188..63da5d15207 100644
--- a/litellm/proxy/common_utils/registry_read_through.py
+++ b/litellm/proxy/common_utils/registry_read_through.py
@@ -125,6 +125,7 @@ async def _resync_model_deployments(model_name: str) -> bool:
         )
         return proxy_server.llm_router is not None
     async with proxy_server.MODEL_RECONCILE_LOCK:
+        await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
         proxy_server.proxy_config._add_deployment(db_models=rows)
         proxy_server.llm_model_list = router.get_model_list()
     return True
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 32b6b841af7..1d6baf5fded 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -7109,11 +7109,10 @@ class ProxyConfig:
                 ],
             )
 
-            # Only load models from DB if "models" is in supported_db_objects (or if supported_db_objects is not set)
-            if self._should_load_db_object(object_type="models"):
-                new_models: Final = await self._get_models_from_db(prisma_client=prisma_client)
-
-                # update llm router
+            load_models: Final = self._should_load_db_object(object_type="models")
+            new_models: Final = await self._get_models_from_db(prisma_client=prisma_client) if load_models else None
+            await self.get_credentials(prisma_client=prisma_client)
+            if load_models:
                 still_desired_ids = await self._update_llm_router(
                     new_models=new_models, proxy_logging_obj=proxy_logging_obj
                 )
@@ -7153,12 +7152,9 @@ class ProxyConfig:
         async def _resync_config_from_db() -> None:
             await self.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
 
-        async def _resync_credentials_from_db() -> None:
-            await self.get_credentials(prisma_client=prisma_client)
-
         subscriber: Final = ConfigSyncSubscriber(
             redis_cache=redis_cache,
-            resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db),
+            resync_callbacks=(_resync_config_from_db,),
         )
         self.config_sync_subscriber = subscriber
         subscriber.start()
@@ -8013,7 +8009,7 @@ class ProxyConfig:
 
     async def get_credentials(self, prisma_client: PrismaClient):
         try:
-            credentials = await CredentialsRepository(prisma_client).find_all()
+            credentials = await CredentialsRepository(WriterPinnedClient(prisma_client.db)).find_all()
             credentials = [self.decrypt_credentials(cred) for cred in credentials]
             await self.delete_credentials(credentials)  # delete credentials that are not in the all-up list
             CredentialAccessor.upsert_credentials(credentials)  # upsert credentials that are in the all-up list
@@ -9597,19 +9593,6 @@ class ProxyStartupEvent:
         )
 
         if store_model_in_db is True:
-            ### GET STORED CREDENTIALS ###
-            scheduler.add_job(
-                proxy_config.get_credentials,
-                "interval",
-                seconds=config_reload_interval_seconds,
-                # REMOVED jitter parameter - major cause of memory leak
-                args=[prisma_client],
-                id="get_credentials_job",
-                replace_existing=True,
-                misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
-            )
-            await proxy_config.get_credentials(prisma_client=prisma_client)
-
             # MEMORY LEAK FIX: Increase interval from 10s to 30s minimum
             # Frequent polling was causing excessive memory allocations
             scheduler.add_job(
@@ -9623,7 +9606,7 @@ class ProxyStartupEvent:
                 misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
             )
 
-            # this will load all existing models on proxy startup
+            # this will load all existing credentials and models on proxy startup
             await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
 
             proxy_config.start_config_sync_subscriber(
diff --git a/litellm/utils.py b/litellm/utils.py
index d0e11bc9551..141ec323776 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -672,11 +672,19 @@ def load_credentials_from_list(kwargs: dict):
     CredentialAccessor: Final = getattr(sys.modules[__name__], "CredentialAccessor")
 
     credential_name: Final = kwargs.get("litellm_credential_name")
-    if credential_name and litellm.credential_list:
-        credential_accessor: Final[Mapping[str, object]] = CredentialAccessor.get_credential_values(credential_name)
-        for key, value in credential_accessor.items():
-            if key not in kwargs:
-                kwargs[key] = value
+    if not credential_name:
+        return
+    credential: Final = CredentialAccessor.find_credential(credential_name)
+    if credential is None:
+        verbose_logger.warning(
+            "litellm_credential_name=%s matched none of the %d loaded credentials; the request runs without it",
+            credential_name,
+            len(litellm.credential_list),
+        )
+        return
+    for key, value in credential.credential_values.items():
+        if key not in kwargs:
+            kwargs[key] = value
 
 
 def get_dynamic_callbacks(
diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py
index 3bb3f75b9b8..2a0ebf9492a 100644
--- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py
+++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py
@@ -824,7 +824,7 @@ class _StopFailingSubscriber(ConfigSyncSubscriber):
         raise RuntimeError("stop failed")
 
 
-async def test_proxy_config_subscriber_resyncs_deployments_and_credentials() -> None:
+async def test_proxy_config_subscriber_resyncs_deployments_only() -> None:
     from litellm.proxy.proxy_server import ProxyConfig
 
     cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()]))
@@ -852,10 +852,7 @@ async def test_proxy_config_subscriber_resyncs_deployments_and_credentials() ->
         await callback()
     await config.stop_config_sync_subscriber()
 
-    assert calls == [
-        ("add_deployment", prisma_client, proxy_logging_obj),
-        ("get_credentials", prisma_client, None),
-    ]
+    assert calls == [("add_deployment", prisma_client, proxy_logging_obj)]
     assert config.config_sync_subscriber is None
     assert subscriber._task is None
 
diff --git a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
index f0fbdea4e85..6f7c20166c5 100644
--- a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
+++ b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
@@ -393,6 +393,51 @@ async def test_resync_model_deployments_mutates_router_under_model_reconcile_loc
     assert not proxy_server.MODEL_RECONCILE_LOCK.locked()
 
 
+@pytest.mark.asyncio
+async def test_resync_model_deployments_loads_db_credentials_before_reconciling_models(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
+    from litellm.types.utils import CredentialItem
+
+    rows: Final = [MagicMock()]
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=rows)
+    router: Final = MagicMock()
+    router.get_model_list.return_value = []
+    installed: Final = MagicMock()
+
+    async def load_credentials_from_db(prisma_client: object) -> None:
+        CredentialAccessor.upsert_credentials(
+            [
+                CredentialItem(
+                    credential_name="openai-cred",
+                    credential_values={"api_key": "sk-from-db"},
+                    credential_info={},
+                )
+            ]
+        )
+
+    def install_models(db_models: object) -> None:
+        installed(db_models=db_models, credential=CredentialAccessor.get_credential_values("openai-cred"))
+
+    monkeypatch.setattr(litellm, "credential_list", [])
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", None)
+    monkeypatch.setattr(proxy_server.proxy_config, "get_credentials", load_credentials_from_db)
+    monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", install_models)
+
+    assert await _resync_model_deployments("model-created-on-a-sibling-replica") is True
+    installed.assert_called_once_with(db_models=rows, credential={"api_key": "sk-from-db"})
+
+
 @pytest.mark.asyncio
 async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch):
     from unittest.mock import AsyncMock, MagicMock
diff --git a/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py b/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py
index ca4d62737b6..54e0aa74a25 100644
--- a/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py
+++ b/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py
@@ -23,7 +23,7 @@ from litellm.proxy.common_utils.scheduled_job_stagger import (
 )
 
 OPERATOR_CRON_JOB_ID = "spend_log_cleanup_job"
-SHARED_INTERVAL_JOB_IDS = ("periodic_reload_job", "get_credentials_job", "add_deployment_job")
+SHARED_INTERVAL_JOB_IDS = ("periodic_reload_job", "add_deployment_job")
 
 
 async def _noop() -> None: ...
diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
index 770cec1834e..ae4ec4086bb 100644
--- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
+++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
@@ -2939,6 +2939,129 @@ async def test_ProxyConfig_add_deployment_applies_db_router_settings(monkeypatch
     fake_router.update_settings.assert_called_once_with(routing_strategy="latency-based-routing")
 
 
+def _stub_add_deployment_collaborators(
+    monkeypatch: pytest.MonkeyPatch, pc: ProxyConfig, fake_prisma: MagicMock
+) -> None:
+    from litellm.proxy import proxy_server
+
+    fake_router = MagicMock()
+    fake_router.get_model_list = MagicMock(return_value=[])
+
+    async def fake_get_config(*args: object, **kwargs: object) -> dict[str, object]:
+        return {}
+
+    monkeypatch.setattr(litellm, "credential_list", [])
+    monkeypatch.setattr(pc, "get_config", fake_get_config)
+    monkeypatch.setattr(pc, "_init_non_llm_objects_in_db", AsyncMock())
+    monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock())
+    monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=None))
+    monkeypatch.setattr(proxy_server, "llm_router", fake_router)
+    monkeypatch.setattr(proxy_server, "master_key", "sk-master")
+    monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server, "proxy_config", pc)
+    monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
+
+
+def _encrypted_credential_row(credential_name: str, api_key: str) -> dict[str, object]:
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
+
+    return {
+        "credential_name": credential_name,
+        "credential_values": {"api_key": encrypt_value_helper(api_key, new_encryption_key="sk-master")},
+        "credential_info": {"custom_llm_provider": "openai"},
+    }
+
+
+def _fake_prisma_with_encrypted_credential(credential_name: str, api_key: str) -> MagicMock:
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_credentialstable.find_many = AsyncMock(
+        return_value=[_encrypted_credential_row(credential_name, api_key)]
+    )
+    return fake_prisma
+
+
+@pytest.mark.asyncio
+async def test_ProxyConfig_add_deployment_loads_db_credentials_before_reconciling_models(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy import proxy_server
+    from litellm.utils import load_credentials_from_list
+
+    pc = ProxyConfig()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_credentialstable.find_many = AsyncMock(return_value=[])
+    _stub_add_deployment_collaborators(monkeypatch, pc, fake_prisma)
+    monkeypatch.setattr(proxy_server, "general_settings", {})
+    installed = MagicMock()
+
+    async def read_models_while_a_credential_lands(prisma_client: object) -> list[MagicMock]:
+        fake_prisma.db.litellm_credentialstable.find_many.return_value = [
+            _encrypted_credential_row("openai-cred", "sk-from-db")
+        ]
+        return [MagicMock()]
+
+    async def install_models(new_models: object, proxy_logging_obj: object) -> None:
+        installed(credential=CredentialAccessor.get_credential_values("openai-cred"))
+
+    monkeypatch.setattr(pc, "_get_models_from_db", read_models_while_a_credential_lands)
+    monkeypatch.setattr(pc, "_update_llm_router", install_models)
+
+    await pc.add_deployment(prisma_client=fake_prisma, proxy_logging_obj=MagicMock())
+
+    installed.assert_called_once_with(credential={"api_key": "sk-from-db"})
+    assert CredentialAccessor.get_credential_values("openai-cred") == {"api_key": "sk-from-db"}
+    request_kwargs = {"litellm_credential_name": "openai-cred"}
+    load_credentials_from_list(request_kwargs)
+    assert request_kwargs == {"litellm_credential_name": "openai-cred", "api_key": "sk-from-db"}
+
+
+@pytest.mark.asyncio
+async def test_ProxyConfig_add_deployment_loads_db_credentials_even_when_models_are_not_db_objects(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy import proxy_server
+
+    pc = ProxyConfig()
+    fake_prisma = _fake_prisma_with_encrypted_credential("openai-cred", "sk-from-db")
+    _stub_add_deployment_collaborators(monkeypatch, pc, fake_prisma)
+    monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["mcp"]})
+    models_fetch = AsyncMock(return_value=[])
+    monkeypatch.setattr(pc, "_get_models_from_db", models_fetch)
+
+    await pc.add_deployment(prisma_client=fake_prisma, proxy_logging_obj=MagicMock())
+
+    models_fetch.assert_not_awaited()
+    assert CredentialAccessor.get_credential_values("openai-cred") == {"api_key": "sk-from-db"}
+
+
+@pytest.mark.asyncio
+async def test_ProxyConfig_get_credentials_reads_from_writer_not_replica(monkeypatch: pytest.MonkeyPatch) -> None:
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy.db.prisma_client import PrismaWrapper
+    from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
+
+    pc = ProxyConfig()
+    writer_inner = MagicMock(name="writer_prisma")
+    reader_inner = MagicMock(name="reader_prisma")
+    writer_inner.litellm_credentialstable.find_many = AsyncMock(
+        return_value=[_encrypted_credential_row("openai-cred", "sk-from-writer")]
+    )
+    reader_inner.litellm_credentialstable.find_many = AsyncMock(return_value=[])
+    fake_prisma = MagicMock()
+    fake_prisma.db = RoutingPrismaWrapper(
+        writer=PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False),
+        reader=PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False),
+    )
+    _stub_add_deployment_collaborators(monkeypatch, pc, fake_prisma)
+
+    await pc.get_credentials(prisma_client=fake_prisma)
+
+    assert CredentialAccessor.get_credential_values("openai-cred") == {"api_key": "sk-from-writer"}
+    reader_inner.litellm_credentialstable.find_many.assert_not_awaited()
+
+
 # ---------------------------------------------------------------------------
 # ProxyConfig._add_general_settings_from_db_config
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index b7bb58378d4..937e4f15741 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -822,16 +822,16 @@ def _mock_scheduled_proxy_config() -> MagicMock:
 
 
 @pytest.mark.asyncio
-async def test_initialize_scheduled_jobs_credentials(monkeypatch):
-    """
-    Test that get_credentials is only called when store_model_in_db is True
-    """
+async def test_initialize_scheduled_jobs_loads_credentials_only_through_add_deployment(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     monkeypatch.delenv("DISABLE_PRISMA_SCHEMA_UPDATE", raising=False)
     monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
+    from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
     from litellm.proxy.proxy_server import ProxyStartupEvent
     from litellm.proxy.utils import ProxyLogging
 
-    # Mock dependencies
     mock_prisma_client = MagicMock()
     mock_proxy_logging = MagicMock(spec=ProxyLogging)
     mock_proxy_logging.slack_alerting_instance = MagicMock()
@@ -841,25 +841,6 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
     with (
         patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
         patch("litellm.proxy.proxy_server.store_model_in_db", False),
-    ):  # set store_model_in_db to False
-        # Test when store_model_in_db is False
-        await ProxyStartupEvent.initialize_scheduled_background_jobs(
-            general_settings={},
-            prisma_client=mock_prisma_client,
-            proxy_budget_rescheduler_min_time=1,
-            proxy_budget_rescheduler_max_time=2,
-            proxy_batch_write_at=5,
-            proxy_logging_obj=mock_proxy_logging,
-        )
-
-        # Verify get_credentials was not called
-        mock_proxy_config.get_credentials.assert_not_called()
-
-    # Now test with store_model_in_db = True
-    with (
-        patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
-        patch("litellm.proxy.proxy_server.store_model_in_db", True),
-        patch("litellm.proxy.proxy_server.get_secret_bool", return_value=True),
     ):
         await ProxyStartupEvent.initialize_scheduled_background_jobs(
             general_settings={},
@@ -870,12 +851,31 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
             proxy_logging_obj=mock_proxy_logging,
         )
 
-        # Verify get_credentials was called both directly and scheduled
-        assert mock_proxy_config.get_credentials.call_count == 1  # Direct call
+        mock_proxy_config.get_credentials.assert_not_called()
+        mock_proxy_config.add_deployment.assert_not_called()
 
-        # Verify a scheduled job was added for get_credentials
-        mock_scheduler_calls = [call[0] for call in mock_proxy_config.get_credentials.mock_calls]
-        assert len(mock_scheduler_calls) > 0
+    scheduler = AsyncIOScheduler()
+    try:
+        with (
+            patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),
+            patch("litellm.proxy.proxy_server.AsyncIOScheduler", return_value=scheduler),
+        ):
+            await ProxyStartupEvent.initialize_scheduled_background_jobs(
+                general_settings={},
+                prisma_client=mock_prisma_client,
+                proxy_budget_rescheduler_min_time=1,
+                proxy_budget_rescheduler_max_time=2,
+                proxy_batch_write_at=5,
+                proxy_logging_obj=mock_proxy_logging,
+            )
+
+        assert scheduler.get_job("get_credentials_job") is None
+        assert scheduler.get_job("add_deployment_job") is not None
+        mock_proxy_config.get_credentials.assert_not_called()
+        assert mock_proxy_config.add_deployment.call_count == 1
+    finally:
+        scheduler.shutdown(wait=False)
 
 
 @pytest.mark.asyncio
@@ -924,7 +924,7 @@ async def test_periodic_reload_job_scheduled_without_store_model_in_db(monkeypat
 @pytest.mark.asyncio
 async def test_initialize_scheduled_jobs_uses_configured_config_reload_interval(monkeypatch):
     """
-    The DB config-reload jobs (add_deployment, get_credentials) that keep multi-pod
+    The DB config-reload job (add_deployment) that keeps multi-pod
     deployments in sync must be scheduled at the configured
     proxy_config_reload_interval_seconds, not a hardcoded value.
     """
@@ -967,7 +967,7 @@ async def test_initialize_scheduled_jobs_uses_configured_config_reload_interval(
         if "id" in job_call.kwargs
     }
     assert scheduled_seconds["add_deployment_job"] == configured_interval
-    assert scheduled_seconds["get_credentials_job"] == configured_interval
+    assert "get_credentials_job" not in scheduled_seconds
 
 
 @pytest.mark.asyncio
@@ -1011,7 +1011,7 @@ async def test_initialize_scheduled_jobs_rejects_non_positive_config_reload_inte
         if "id" in job_call.kwargs
     }
     assert scheduled_seconds["add_deployment_job"] == 30
-    assert scheduled_seconds["get_credentials_job"] == 30
+    assert "get_credentials_job" not in scheduled_seconds
 
 
 @pytest.mark.asyncio
@@ -7446,10 +7446,8 @@ async def test_store_model_in_db_db_override_when_config_false():
         # store_model_in_db should now be True (overridden by DB)
         assert ps.store_model_in_db is True
 
-        # add_deployment and get_credentials should have been called
-        # since store_model_in_db is now True
         assert mock_proxy_config.add_deployment.call_count == 1
-        assert mock_proxy_config.get_credentials.call_count == 1
+        mock_proxy_config.get_credentials.assert_not_called()
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 8a56a84ade7..4ce68b71079 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -6092,3 +6092,47 @@ class TestFinalOptionalParamsLineRedaction:
 
         assert "'max_tokens': 17" in printed
         assert "'temperature': 0.25" in printed
+
+
+def _credential_warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
+    return [record.getMessage() for record in caplog.records if "litellm_credential_name=" in record.getMessage()]
+
+
+def test_load_credentials_from_list_warns_when_the_named_credential_is_not_loaded(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    from litellm.utils import load_credentials_from_list
+
+    monkeypatch.setattr(litellm, "credential_list", [])
+    request_kwargs = {"litellm_credential_name": "openai-cred", "model": "openai/gpt-5.4-mini"}
+    with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+        load_credentials_from_list(request_kwargs)
+
+    assert request_kwargs == {"litellm_credential_name": "openai-cred", "model": "openai/gpt-5.4-mini"}
+    assert _credential_warnings(caplog) == [
+        "litellm_credential_name=openai-cred matched none of the 0 loaded credentials; the request runs without it"
+    ]
+
+
+def test_load_credentials_from_list_fills_kwargs_from_the_loaded_credential_without_warning(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    from litellm.types.utils import CredentialItem
+    from litellm.utils import load_credentials_from_list
+
+    loaded = CredentialItem(
+        credential_name="openai-cred",
+        credential_values={"api_key": "sk-from-db", "api_base": "https://credential.example"},
+        credential_info={},
+    )
+    monkeypatch.setattr(litellm, "credential_list", [loaded])
+    request_kwargs = {"litellm_credential_name": "openai-cred", "api_base": "https://request.example"}
+    with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+        load_credentials_from_list(request_kwargs)
+
+    assert request_kwargs == {
+        "litellm_credential_name": "openai-cred",
+        "api_base": "https://request.example",
+        "api_key": "sk-from-db",
+    }
+    assert _credential_warnings(caplog) == []

From f769aa4675de225a04fdc055c092318d94612502 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:11:20 -0700
Subject: [PATCH 17/27] fix(router): give cooldowns their own cache so siblings
 see a bench in ~1s (#40025)

Cooldown entries rode the router-wide DualCache, which re-reads a key that is
missing from memory at most once every 10s. A deployment benched on one replica
therefore kept taking traffic on its siblings for up to 10 seconds, and the same
shared in-memory tier could evict a live cooldown once 200 unrelated router keys
crowded it out, which sent even the benching replica back to the dead deployment.

CooldownCache now owns a DualCache over the router's Redis with a 1s read
interval and an in-memory tier that only holds cooldown keys. Redis is attached
lazily because the router builds the cooldown cache before it wires Redis up.
---
 litellm/constants.py                          |   3 +
 litellm/router_utils/cooldown_cache.py        |  48 +++--
 .../test_router_cooldown_per_deployment.py    |  16 +-
 .../router_utils/test_cooldown_cache.py       | 184 ++++++++++++++++--
 4 files changed, 212 insertions(+), 39 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index defc9337e9b..8ef9523a60a 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -73,6 +73,9 @@ DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096))
 DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3))
 DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1))
 DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5))
+DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = float(
+    os.getenv("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", "1")
+)
 DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5))
 DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1))
 DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py
index 9e7f457f631..ef29f7d8fd3 100644
--- a/litellm/router_utils/cooldown_cache.py
+++ b/litellm/router_utils/cooldown_cache.py
@@ -12,6 +12,7 @@ from typing_extensions import TypedDict
 from litellm import verbose_logger
 from litellm.caching.caching import DualCache
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.constants import DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS
 from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 
 if TYPE_CHECKING:
@@ -36,10 +37,19 @@ _MAX_CORRECTED_IN_MEMORY_TTL_SECONDS: Final = 60.0
 
 
 class CooldownCache:
-    def __init__(self, cache: DualCache, default_cooldown_time: float):
+    def __init__(
+        self,
+        cache: DualCache,
+        default_cooldown_time: float,
+        redis_read_interval_seconds: float = DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS,
+    ):
         self.cache = cache
         self.default_cooldown_time = default_cooldown_time
         self.in_memory_cache = InMemoryCache()
+        self._cooldown_store = DualCache(
+            in_memory_cache=self.in_memory_cache,
+            default_redis_batch_cache_expiry=redis_read_interval_seconds,
+        )
         # Initialize the masker with custom settings for exception strings
         self.exception_masker = SensitiveDataMasker(
             visible_prefix=50,  # Show first 50 characters
@@ -48,6 +58,21 @@ class CooldownCache:
             mask_short_values=False,  # Truncate long messages only; keep short ones readable
         )
 
+    @property
+    def cooldown_store(self) -> DualCache:
+        """
+        The cache cooldown entries live in, with the router's Redis attached on first use.
+
+        It is kept separate from the router-wide cache so that a key missing from memory is
+        re-read from Redis every `redis_read_interval_seconds` rather than on the router
+        cache's much longer batch interval, which is what lets a sibling replica see a
+        cooldown another replica wrote, and so that unrelated router keys cannot evict a
+        cooldown from the in-memory tier before it expires. Redis is attached lazily because
+        the router builds its cooldown cache before it wires up the shared Redis client.
+        """
+        self._cooldown_store.attach_redis_cache(self.cache.redis_cache)
+        return self._cooldown_store
+
     def _common_add_cooldown_logic(
         self, model_id: str, original_exception, exception_status, cooldown_time: float
     ) -> tuple[str, CooldownCacheValue]:
@@ -93,7 +118,7 @@ class CooldownCache:
             )
 
             # Set the cache with a TTL equal to the cooldown time
-            self.cache.set_cache(
+            self.cooldown_store.set_cache(
                 value=cooldown_data,
                 key=cooldown_key,
                 ttl=_cooldown_time,
@@ -122,13 +147,13 @@ class CooldownCache:
         cooldown_cache_value: Final = CooldownCacheValue(**result)  # pyright: ignore[reportUnknownArgumentType] - result comes from an untyped cache read, not from our own code
         remaining: Final = (cooldown_cache_value["timestamp"] + cooldown_cache_value["cooldown_time"]) - current_time
         if remaining <= 0:
-            self.cache.in_memory_cache.delete_cache(key)
+            self.in_memory_cache.delete_cache(key)
             return None
-        current_expiry: Final = self.cache.in_memory_cache.ttl_dict.get(key)
+        current_expiry: Final = self.in_memory_cache.ttl_dict.get(key)
         if current_expiry is not None and current_expiry > current_time + remaining + 5:
             corrected_ttl: Final = min(remaining, _MAX_CORRECTED_IN_MEMORY_TTL_SECONDS)
-            self.cache.in_memory_cache.delete_cache(key)
-            self.cache.in_memory_cache.set_cache(key, result, ttl=corrected_ttl)
+            self.in_memory_cache.delete_cache(key)
+            self.in_memory_cache.set_cache(key, result, ttl=corrected_ttl)
         return cooldown_cache_value
 
     async def async_get_active_cooldowns(
@@ -137,12 +162,7 @@ class CooldownCache:
         # Generate the keys for the deployments
         keys: Final = [CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids]
 
-        # Retrieve the values for the keys using mget
-        ## more likely to be none if no models ratelimited. So just check redis every 1s
-        ## each redis call adds ~100ms latency.
-
-        ## check in memory cache first
-        results: Final = await self.cache.async_batch_get_cache(keys=keys, parent_otel_span=parent_otel_span)
+        results: Final = await self.cooldown_store.async_batch_get_cache(keys=keys, parent_otel_span=parent_otel_span)
         active_cooldowns: Final[list[tuple[str, CooldownCacheValue]]] = []
 
         if results is None or all(v is None for v in results):
@@ -164,7 +184,7 @@ class CooldownCache:
         # Generate the keys for the deployments
         keys: Final = [CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids]
         # Retrieve the values for the keys using mget
-        results: Final = self.cache.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
+        results: Final = self.cooldown_store.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
 
         active_cooldowns: Final = []
         current_time: Final = time.time()
@@ -184,7 +204,7 @@ class CooldownCache:
         keys: Final = [f"deployment:{model_id}:cooldown" for model_id in model_ids]
 
         # Retrieve the values for the keys using mget
-        results: Final = self.cache.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
+        results: Final = self.cooldown_store.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
 
         min_cooldown_time: float | None = None
         # Process the results
diff --git a/tests/router_unit_tests/test_router_cooldown_per_deployment.py b/tests/router_unit_tests/test_router_cooldown_per_deployment.py
index b8ae8a8c013..228dfed38b9 100644
--- a/tests/router_unit_tests/test_router_cooldown_per_deployment.py
+++ b/tests/router_unit_tests/test_router_cooldown_per_deployment.py
@@ -246,12 +246,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired cooldown entry must not appear in active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
+        assert cc.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
 
     def test_active_entry_is_returned(self):
         """
@@ -267,7 +267,7 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time(),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, active_value, ttl=60)
+        cc.in_memory_cache.set_cache(key, active_value, ttl=60)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
@@ -290,14 +290,14 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - (60.0 - remaining),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, value, ttl=600)
+        cc.in_memory_cache.set_cache(key, value, ttl=600)
 
-        before_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        before_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert before_expiry is not None
 
         cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
-        after_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        after_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert after_expiry is not None
         corrected_remaining = after_expiry - time.time()
         assert corrected_remaining <= 60.0, "Corrected TTL must not exceed 60s"
@@ -318,12 +318,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = await cc.async_get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired entry must not appear in async active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None
+        assert cc.in_memory_cache.get_cache(key) is None
 
 
 class TestFallbackDeploymentCooldown:
diff --git a/tests/test_litellm/router_utils/test_cooldown_cache.py b/tests/test_litellm/router_utils/test_cooldown_cache.py
index 68e9aeaa4fc..6f90fa8465f 100644
--- a/tests/test_litellm/router_utils/test_cooldown_cache.py
+++ b/tests/test_litellm/router_utils/test_cooldown_cache.py
@@ -268,12 +268,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired cooldown entry must not appear in active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
+        assert cc.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
 
     def test_active_entry_is_returned(self):
         """
@@ -289,7 +289,7 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time(),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, active_value, ttl=60)
+        cc.in_memory_cache.set_cache(key, active_value, ttl=60)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
@@ -312,14 +312,14 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - (60.0 - remaining),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, value, ttl=600)
+        cc.in_memory_cache.set_cache(key, value, ttl=600)
 
-        before_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        before_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert before_expiry is not None
 
         cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
-        after_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        after_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert after_expiry is not None
         corrected_remaining = after_expiry - time.time()
         assert corrected_remaining <= 60.0, "Corrected TTL must not exceed 60s"
@@ -340,12 +340,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = await cc.async_get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired entry must not appear in async active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None
+        assert cc.in_memory_cache.get_cache(key) is None
 
     @pytest.mark.asyncio
     async def test_async_active_entry_is_returned(self):
@@ -363,7 +363,7 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time(),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, active_value, ttl=60)
+        cc.in_memory_cache.set_cache(key, active_value, ttl=60)
 
         active = await cc.async_get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
@@ -389,18 +389,18 @@ class TestCorrectedActiveCooldown:
         cc = self._make_cooldown_cache()
         key = "deployment:expired-dep:cooldown"
         entry = self._entry(timestamp=time.time() - 120.0, cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=600)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=600)
 
         result = cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
         assert result is None
-        assert cc.cache.in_memory_cache.get_cache(key) is None
+        assert cc.in_memory_cache.get_cache(key) is None
 
     def test_active_entry_within_window_returns_value(self):
         cc = self._make_cooldown_cache()
         key = "deployment:active-dep:cooldown"
         entry = self._entry(timestamp=time.time(), cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=60)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=60)
 
         result = cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
@@ -412,12 +412,12 @@ class TestCorrectedActiveCooldown:
         key = "deployment:backfilled-dep:cooldown"
         remaining = 30.0
         entry = self._entry(timestamp=time.time() - (60.0 - remaining), cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=600)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=600)
 
         result = cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
         assert result is not None
-        corrected_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        corrected_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert corrected_expiry is not None
         assert corrected_expiry - time.time() <= 60.0
 
@@ -425,10 +425,160 @@ class TestCorrectedActiveCooldown:
         cc = self._make_cooldown_cache()
         key = "deployment:normal-dep:cooldown"
         entry = self._entry(timestamp=time.time(), cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=60)
-        original_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=60)
+        original_expiry = cc.in_memory_cache.ttl_dict.get(key)
 
         cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
-        after_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        after_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert after_expiry == original_expiry
+
+
+class SharedRedisDouble:
+    """
+    In-process stand-in for RedisCache, shared by several DualCache instances so that
+    tests can model two proxy replicas talking to one Redis.
+    """
+
+    def __init__(self) -> None:
+        self.store: dict = {}  # mutable-ok: stands in for Redis' own mutable keyspace
+
+    def set_cache(self, key, value, **kwargs):
+        self.store[key] = value
+
+    async def async_set_cache(self, key, value, **kwargs):
+        self.store[key] = value
+
+    def batch_get_cache(self, key_list, parent_otel_span=None, **kwargs):
+        return {key: self.store.get(key) for key in key_list}
+
+    async def async_batch_get_cache(self, key_list, parent_otel_span=None, **kwargs):
+        return {key: self.store.get(key) for key in key_list}
+
+
+class TestCooldownPropagationBetweenReplicas:
+    """
+    A cooldown written by one replica has to reach its siblings quickly. The router's own
+    DualCache re-reads a key that is missing from memory only every 10s, so cooldown reads
+    get their own cache with a much shorter Redis read interval.
+    """
+
+    def _make_replica(self, redis: SharedRedisDouble, read_interval: float | None = None) -> CooldownCache:
+        router_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis)
+        if read_interval is None:
+            return CooldownCache(cache=router_cache, default_cooldown_time=60.0)
+        return CooldownCache(
+            cache=router_cache,
+            default_cooldown_time=60.0,
+            redis_read_interval_seconds=read_interval,
+        )
+
+    @pytest.mark.asyncio
+    async def test_sibling_replica_sees_cooldown_within_configured_read_interval(self):
+        redis = SharedRedisDouble()
+        replica_a = self._make_replica(redis, read_interval=0.25)
+        replica_b = self._make_replica(redis, read_interval=0.25)
+        model_id = "shared-deployment"
+
+        assert await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None) == []
+
+        replica_a.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        time.sleep(0.3)
+
+        active = await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "sibling replica must pick up a cooldown written by another replica within the read interval"
+        )
+
+    @pytest.mark.asyncio
+    async def test_sibling_replica_sees_cooldown_within_default_read_interval(self):
+        redis = SharedRedisDouble()
+        replica_a = self._make_replica(redis)
+        replica_b = self._make_replica(redis)
+        model_id = "default-interval-deployment"
+
+        assert await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None) == []
+
+        replica_a.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        time.sleep(1.2)
+
+        active = await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "the shipped default read interval must let a sibling replica see a cooldown about a second later"
+        )
+
+    def test_sync_read_path_sees_sibling_cooldown_within_read_interval(self):
+        redis = SharedRedisDouble()
+        replica_a = self._make_replica(redis, read_interval=0.25)
+        replica_b = self._make_replica(redis, read_interval=0.25)
+        model_id = "sync-shared-deployment"
+
+        assert replica_b.get_active_cooldowns([model_id], parent_otel_span=None) == []
+
+        replica_a.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        time.sleep(0.3)
+
+        active = replica_b.get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active]
+
+    @pytest.mark.asyncio
+    async def test_redis_attached_after_construction_is_still_used(self):
+        redis = SharedRedisDouble()
+        router_cache = DualCache(in_memory_cache=InMemoryCache())
+        writer = CooldownCache(cache=router_cache, default_cooldown_time=60.0, redis_read_interval_seconds=0.25)
+        router_cache.attach_redis_cache(redis)
+        reader = self._make_replica(redis, read_interval=0.25)
+        model_id = "late-redis-deployment"
+
+        writer.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        active = await reader.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "a router that wires Redis after building its cooldown cache must still publish cooldowns to it"
+        )
+
+
+class TestCooldownSurvivesUnrelatedCacheTraffic:
+    @pytest.mark.asyncio
+    async def test_unrelated_router_cache_writes_do_not_evict_active_cooldown(self):
+        router_cache = DualCache(in_memory_cache=InMemoryCache())
+        cc = CooldownCache(cache=router_cache, default_cooldown_time=60.0)
+        model_id = "busy-router-deployment"
+
+        cc.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=30.0,
+        )
+
+        for i in range(400):
+            router_cache.set_cache(key=f"unrelated-router-key-{i}", value={"n": i})
+
+        active = await cc.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "unrelated router cache traffic must not evict a cooldown that is still running"
+        )

From 3448175184d8b68dbcf9ec007d62f85651bb0f41 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 11:44:49 -0700
Subject: [PATCH 18/27] fix(responses bridge): keep mid-conversation system
 messages in input

Only the leading run of system messages (before the first non-system
message) is joined into the Responses `instructions` field. A system
message that arrives after a user, assistant, or tool turn now becomes a
system input item at its position, whether its content is a string or a
list, so a client that re-sends the same reminder as a string on the
next request produces byte-identical input and `instructions` stays
stable. Claude Code >= 2.1.237 appends such reminders after every user
turn, and folding them into `instructions` made Azure treat every
request as a cold prompt (cached_tokens 0 on every request of a
session).

Fixes #40198
---
 .../transformation.py                         |   9 +-
 ...responses_transformation_transformation.py | 140 ++++++++++++++++++
 2 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py
index 87350b5479c..4fe069b0b7d 100644
--- a/litellm/completion_extras/litellm_responses_transformation/transformation.py
+++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py
@@ -370,7 +370,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             and isinstance(tool_call.get("custom"), dict)
         )
 
-        for msg in messages:
+        leading_system_count: Final = next(
+            (index for index, msg in enumerate(messages) if msg.get("role") != "system"),
+            len(messages),
+        )
+
+        for index, msg in enumerate(messages):
             role = msg.get("role")
             content = msg.get("content", "")
             tool_calls = msg.get("tool_calls")
@@ -378,7 +383,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
             if role == "system":
                 # Extract system message as instructions
-                if isinstance(content, str):
+                if isinstance(content, str) and index < leading_system_count:
                     if instructions:
                         # Concatenate multiple system prompts with a space
                         instructions = f"{instructions} {content}"
diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
index d4d47b145d1..e4ada0a9b31 100644
--- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
+++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
@@ -4147,3 +4147,143 @@ def test_streaming_final_chunk_carries_provider_metadata():
     assert chunks[-1]["content_filters"] == content_filters
     assert "background" not in chunks[-1]
     assert all("service_tier" not in chunk for chunk in chunks[:-1])
+
+
+def _system_input_item(text: str) -> dict[str, object]:
+    return {"type": "message", "role": "system", "content": [{"type": "input_text", "text": text}]}
+
+
+def test_mid_conversation_system_string_stays_in_input_after_a_user_turn():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Read the file."},
+            {"role": "system", "content": "<total_tokens>14982391 tokens left</total_tokens>"},
+            {"role": "user", "content": "Now summarize it."},
+        ]
+    )
+
+    assert instructions == "You are a helpful assistant."
+    assert input_items == [
+        {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read the file."}]},
+        _system_input_item("<total_tokens>14982391 tokens left</total_tokens>"),
+        {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Now summarize it."}]},
+    ]
+
+
+def test_leading_system_strings_still_join_instructions_without_a_following_turn():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "system", "content": "Be brief."},
+            {"role": "system", "content": "Answer in French."},
+        ]
+    )
+
+    assert instructions == "Be brief. Answer in French."
+    assert input_items == []
+
+
+def test_mid_conversation_system_reminder_as_string_and_as_text_block_produce_identical_input_items():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+    reminder: Final = "<total_tokens>14982391 tokens left</total_tokens>"
+
+    as_string, string_instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [{"role": "user", "content": "Read the file."}, {"role": "system", "content": reminder}]
+    )
+    as_block, block_instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "user", "content": "Read the file."},
+            {
+                "role": "system",
+                "content": [{"type": "text", "text": reminder, "cache_control": {"type": "ephemeral"}}],
+            },
+        ]
+    )
+
+    assert string_instructions is None
+    assert block_instructions is None
+    assert json.dumps(as_string) == json.dumps(as_block)
+    assert as_string[1] == _system_input_item(reminder)
+
+
+def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requests():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+    top_level_system: Final = [{"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}]
+    first_reminder: Final = "<system-reminder>27k chars of deferred tools</system-reminder>"
+    second_reminder: Final = "<total_tokens>14982391 tokens left</total_tokens>"
+    first_request_messages: Final = [
+        {"role": "system", "content": top_level_system},
+        {"role": "user", "content": "Read inventory.py."},
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": first_reminder, "cache_control": {"type": "ephemeral"}}],
+        },
+    ]
+    second_request_messages: Final = [
+        {"role": "system", "content": top_level_system},
+        {"role": "user", "content": "Read inventory.py."},
+        {"role": "system", "content": first_reminder},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [
+                {
+                    "id": "call_1",
+                    "type": "function",
+                    "function": {"name": "Read", "arguments": '{"file_path": "inventory.py"}'},
+                }
+            ],
+        },
+        {"role": "tool", "tool_call_id": "call_1", "content": "ITEMS = []"},
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": second_reminder, "cache_control": {"type": "ephemeral"}}],
+        },
+    ]
+
+    first_request: Final = handler.transform_request(
+        model="gpt-5.6-luna",
+        messages=first_request_messages,
+        optional_params={},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+    second_request: Final = handler.transform_request(
+        model="gpt-5.6-luna",
+        messages=second_request_messages,
+        optional_params={},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert "instructions" not in first_request
+    assert "instructions" not in second_request
+    assert first_request["input"][0] == _system_input_item("You are Claude Code.")
+    assert json.dumps(second_request["input"][: len(first_request["input"])]) == json.dumps(first_request["input"])
+    assert second_request["input"][len(first_request["input"]) :] == [
+        {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": '{"file_path": "inventory.py"}'},
+        {"type": "function_call_output", "call_id": "call_1", "output": [{"type": "input_text", "text": "ITEMS = []"}]},
+        _system_input_item(second_reminder),
+    ]
+
+
+def test_system_string_after_a_developer_message_stays_in_input_in_client_order():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "developer", "content": "Always answer in French."},
+            {"role": "system", "content": "Be brief."},
+            {"role": "user", "content": "Bonjour"},
+        ]
+    )
+
+    assert instructions is None
+    assert [item["role"] for item in input_items] == ["developer", "system", "user"]
+    assert input_items[1] == _system_input_item("Be brief.")

From ee1a6407cb5b901660e62d757f83cdd6a99e5a34 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 11:45:02 -0700
Subject: [PATCH 19/27] fix(guardrails): keep guardrail telemetry when a policy
 pipeline blocks or modifies the response (#40211)

* fix(guardrails): keep guardrail telemetry when a policy pipeline blocks or modifies the response

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(guardrails): count every raw-snapshot guardrail evaluation and type the telemetry carry helpers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(policy_engine): type the recording guardrail hooks and telemetry test parameters

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yucheng <yucheng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/policy_engine/pipeline_executor.py  | 104 +++++++++----
 .../policy_engine/test_pipeline_executor.py   | 141 ++++++++++++++++++
 2 files changed, 216 insertions(+), 29 deletions(-)

diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index 4be0f556ed7..1bcda38657b 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -6,7 +6,7 @@ pass/fail actions (allow, block, next, modify_response) and data forwarding.
 """
 
 import time
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from typing import Any, Final, Literal
 
 import litellm
@@ -16,7 +16,11 @@ from litellm.integrations.custom_guardrail import (
     ModifyResponseException,
 )
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.litellm_core_utils.core_helpers import independent_snapshot
+from litellm.litellm_core_utils.core_helpers import (
+    get_metadata_variable_name_from_kwargs,
+    get_or_create_metadata_bucket,
+    independent_snapshot,
+)
 from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
     UnifiedLLMGuardrails,
 )
@@ -25,6 +29,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import (
     PipelineStep,
     PipelineStepResult,
 )
+from litellm.types.utils import StandardLoggingGuardrailInformation
 
 try:
     from fastapi.exceptions import HTTPException
@@ -118,6 +123,7 @@ class PipelineExecutor:
                 return _allow_result(step_results=step_results, working_data=working_data, request_data=data)
 
             if action == "block":
+                _carry_working_guardrail_information(working_data=working_data, request_data=data)
                 return PipelineExecutionResult(
                     terminal_action="block",
                     step_results=step_results,
@@ -126,6 +132,7 @@ class PipelineExecutor:
                 )
 
             if action == "modify_response":
+                _carry_working_guardrail_information(working_data=working_data, request_data=data)
                 return PipelineExecutionResult(
                     terminal_action="modify_response",
                     step_results=step_results,
@@ -168,34 +175,33 @@ class PipelineExecutor:
             verbose_proxy_logger.warning("Pipeline: guardrail '%s' not found in callbacks", step.guardrail)
             return ("error", None, f"Guardrail '{step.guardrail}' not found", None)
 
+        # Inject guardrail name into metadata so should_run_guardrail() allows it
+        if "metadata" not in data:
+            data["metadata"] = {}
+        data["metadata"]["guardrails"] = [step.guardrail]
+
+        # A scan_raw_request step evaluates the pristine pre-pipeline
+        # snapshot instead of `data` (which earlier pass_data steps in
+        # this same pipeline may have already rewritten), same reason
+        # the normal sequential/parallel guardrail loops do this.
+        scans_raw_request: Final = callback.scan_raw_request
+        hook_input: Final[dict] = (  # mutable-ok: same request-payload shape as data
+            independent_snapshot(raw_request_snapshot)
+            if scans_raw_request and raw_request_snapshot is not None
+            else data
+        )
+        if hook_input is not data:
+            hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
+        snapshot_entries_before: Final = len(_recorded_guardrail_information(hook_input))
+
+        # Use unified_guardrail path if callback implements apply_guardrail
+        target: CustomLogger = callback
+        use_unified: Final = "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
+        if use_unified:
+            hook_input["guardrail_to_apply"] = callback
+            target = UnifiedLLMGuardrails()
+
         try:
-            # Inject guardrail name into metadata so should_run_guardrail() allows it
-            if "metadata" not in data:
-                data["metadata"] = {}
-            data["metadata"]["guardrails"] = [step.guardrail]
-
-            # A scan_raw_request step evaluates the pristine pre-pipeline
-            # snapshot instead of `data` (which earlier pass_data steps in
-            # this same pipeline may have already rewritten), same reason
-            # the normal sequential/parallel guardrail loops do this.
-            scans_raw_request: Final = callback.scan_raw_request
-            hook_input: Final[dict] = (  # mutable-ok: same request-payload shape as data
-                independent_snapshot(raw_request_snapshot)
-                if scans_raw_request and raw_request_snapshot is not None
-                else data
-            )
-            if hook_input is not data:
-                hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
-
-            # Use unified_guardrail path if callback implements apply_guardrail
-            target: CustomLogger = callback
-            use_unified: Final = (
-                "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
-            )
-            if use_unified:
-                hook_input["guardrail_to_apply"] = callback
-                target = UnifiedLLMGuardrails()
-
             if mode == "pre_call":
                 response = await target.async_pre_call_hook(
                     user_api_key_dict=user_api_key_dict,
@@ -233,6 +239,12 @@ class PipelineExecutor:
             else:
                 verbose_proxy_logger.error("Pipeline: unexpected error from guardrail '%s': %s", step.guardrail, e)
                 return ("error", None, str(e), e)
+        finally:
+            if hook_input is not data:
+                _append_guardrail_information(
+                    request_data=data,
+                    entries=_recorded_guardrail_information(hook_input)[snapshot_entries_before:],
+                )
 
     @staticmethod
     def find_guardrail_callback(guardrail_name: str) -> CustomGuardrail | None:
@@ -283,6 +295,40 @@ def _restore_request_guardrails(
     return {**working_data, "metadata": stripped}  # mutable-ok: request dict
 
 
+_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
+
+
+def _recorded_guardrail_information(source: Mapping[str, object]) -> list[StandardLoggingGuardrailInformation]:
+    bucket: Final = source.get(get_metadata_variable_name_from_kwargs(source))
+    recorded: Final = bucket.get(_GUARDRAIL_INFORMATION_KEY) if isinstance(bucket, dict) else None
+    return recorded if isinstance(recorded, list) else []
+
+
+def _append_guardrail_information(
+    request_data: dict[str, object],  # mutable-ok: same request-payload shape as execute_steps' data
+    entries: Sequence[StandardLoggingGuardrailInformation],
+) -> None:
+    if not entries:
+        return
+    _, request_bucket = get_or_create_metadata_bucket(request_data)
+    existing: Final = request_bucket.get(_GUARDRAIL_INFORMATION_KEY)
+    if isinstance(existing, list):
+        existing.extend(entries)
+        return
+    request_bucket[_GUARDRAIL_INFORMATION_KEY] = list(entries)
+
+
+def _carry_working_guardrail_information(
+    working_data: Mapping[str, object],
+    request_data: dict[str, object],  # mutable-ok: same request-payload shape as execute_steps' data
+) -> None:
+    recorded: Final = _recorded_guardrail_information(working_data)
+    existing: Final = _recorded_guardrail_information(request_data)
+    if recorded is existing:
+        return
+    _append_guardrail_information(request_data=request_data, entries=[e for e in recorded if e not in existing])
+
+
 def _pipeline_action_for_outcome(step: PipelineStep, outcome: str) -> str:
     """
     Map pipeline step outcome to the configured action.
diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
index 4fcb7d22588..e95d001bb12 100644
--- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
+++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
@@ -4,20 +4,26 @@ Tests for the pipeline executor.
 Uses mock guardrails to validate pipeline execution without external services.
 """
 
+import copy
+from typing import Literal
 from unittest.mock import MagicMock
 
 import pytest
 
 import litellm
+from litellm.caching.dual_cache import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import (
     CustomCodeGuardrail,
 )
 from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor
+from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.proxy.policy_engine.pipeline_types import (
     GuardrailPipeline,
     PipelineStep,
 )
+from litellm.types.utils import CallTypesLiteral
 
 try:
     from fastapi.exceptions import HTTPException
@@ -158,11 +164,146 @@ class ContentCheckGuardrail(CustomGuardrail):
         return None
 
 
+class RecordingGuardrail(CustomGuardrail):
+    def __init__(self, guardrail_name: str, scan_raw_request: bool = False, block: bool = True):
+        super().__init__(
+            guardrail_name=guardrail_name,
+            event_hook="pre_call",
+            default_on=True,
+            scan_raw_request=scan_raw_request,
+        )
+        self.block = block
+
+    def should_run_guardrail(self, data: dict[str, object], event_type: GuardrailEventHooks) -> bool:
+        return True
+
+    async def async_pre_call_hook(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        cache: DualCache,
+        data: dict[str, object],
+        call_type: CallTypesLiteral,
+    ) -> dict[str, object]:
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_json_response={"detected": ["aws_access_key"]},
+            request_data=data,
+            guardrail_status="guardrail_intervened" if self.block else "success",
+        )
+        if self.block:
+            raise HTTPException(status_code=400, detail="Content policy violation")
+        return copy.deepcopy(data)
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Tests
 # ─────────────────────────────────────────────────────────────────────────────
 
 
+@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
+@pytest.mark.asyncio
+@pytest.mark.parametrize("scan_raw_request", [False, True])
+@pytest.mark.parametrize("on_fail", ["block", "modify_response"])
+async def test_terminal_block_carries_guardrail_information_to_request(
+    monkeypatch: pytest.MonkeyPatch, scan_raw_request: bool, on_fail: Literal["block", "modify_response"]
+):
+    """
+    Spend logging and the Guardrails Monitor read standard_logging_guardrail_information
+    off the caller's request dict. A blocking step records it on the executor's
+    working copy (or the raw-request snapshot), so the terminal result must carry it
+    back onto the request or the block is never counted.
+    """
+    guard = RecordingGuardrail(guardrail_name="credentials-api-keys", scan_raw_request=scan_raw_request)
+    monkeypatch.setattr(litellm, "callbacks", [guard])
+    data = {
+        "messages": [{"role": "user", "content": "key AKIAIOSFODNN7EXAMPLE"}],
+        "metadata": {"user_api_key_hash": "abc"},
+    }
+
+    result = await PipelineExecutor.execute_steps(
+        steps=[PipelineStep(guardrail="credentials-api-keys", on_fail=on_fail, on_pass="next")],
+        mode="pre_call",
+        data=data,
+        user_api_key_dict=MagicMock(),
+        call_type="completion",
+        policy_name="baseline-pii-protection",
+        raw_request_snapshot={"messages": data["messages"], "metadata": {"user_api_key_hash": "abc"}},
+    )
+
+    assert result.terminal_action == on_fail
+    recorded = data["metadata"]["standard_logging_guardrail_information"]
+    assert [entry["guardrail_name"] for entry in recorded] == ["credentials-api-keys"]
+    assert recorded[0]["guardrail_status"] == "guardrail_intervened"
+    assert data["metadata"]["user_api_key_hash"] == "abc"
+    assert "guardrails" not in data["metadata"]
+
+
+@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
+@pytest.mark.asyncio
+async def test_terminal_block_merges_guardrail_information_without_duplicates(monkeypatch: pytest.MonkeyPatch):
+    """A pass_data step that returns a rewritten copy of the request, and a scan_raw_request step
+    that evaluates a deep copy taken before the pipeline ran, both leave earlier entries in two
+    dicts at once. Those must be carried back once while every step's own entry is kept."""
+    first = RecordingGuardrail(guardrail_name="pii-scan", block=False)
+    second = RecordingGuardrail(guardrail_name="credentials-api-keys", scan_raw_request=True)
+    monkeypatch.setattr(litellm, "callbacks", [first, second])
+    earlier = {"guardrail_name": "earlier-guard", "guardrail_status": "success"}
+    data = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}}
+    data["metadata"]["standard_logging_guardrail_information"] = [earlier]
+
+    result = await PipelineExecutor.execute_steps(
+        steps=[
+            PipelineStep(guardrail="pii-scan", on_fail="block", on_pass="next", pass_data=True),
+            PipelineStep(guardrail="credentials-api-keys", on_fail="block", on_pass="next"),
+        ],
+        mode="pre_call",
+        data=data,
+        user_api_key_dict=MagicMock(),
+        call_type="completion",
+        policy_name="baseline-pii-protection",
+        raw_request_snapshot={
+            "messages": data["messages"],
+            "metadata": {"standard_logging_guardrail_information": [dict(earlier)]},
+        },
+    )
+
+    assert result.terminal_action == "block"
+    recorded = data["metadata"]["standard_logging_guardrail_information"]
+    assert [entry["guardrail_name"] for entry in recorded] == ["earlier-guard", "pii-scan", "credentials-api-keys"]
+
+
+@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
+@pytest.mark.asyncio
+async def test_repeated_scan_raw_request_step_is_counted_once_per_evaluation(monkeypatch: pytest.MonkeyPatch):
+    """Running the same raw-scan guardrail twice yields two identical entries; both must reach the caller,
+    while the entries the raw snapshot already held before the pipeline ran are not copied again."""
+    guard = RecordingGuardrail(guardrail_name="credentials-raw", scan_raw_request=True, block=False)
+    monkeypatch.setattr(litellm, "callbacks", [guard])
+    earlier = {"guardrail_name": "earlier-guard", "guardrail_status": "success"}
+    data = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}}
+    data["metadata"]["standard_logging_guardrail_information"] = [earlier]
+
+    result = await PipelineExecutor.execute_steps(
+        steps=[
+            PipelineStep(guardrail="credentials-raw", on_fail="block", on_pass="next"),
+            PipelineStep(guardrail="credentials-raw", on_fail="block", on_pass="next"),
+        ],
+        mode="pre_call",
+        data=data,
+        user_api_key_dict=MagicMock(),
+        call_type="completion",
+        policy_name="raw-scan-policy",
+        raw_request_snapshot={
+            "messages": data["messages"],
+            "metadata": {"standard_logging_guardrail_information": [dict(earlier)]},
+        },
+    )
+
+    assert result.terminal_action == "allow"
+    assert result.modified_data is not None
+    recorded = result.modified_data["metadata"]["standard_logging_guardrail_information"]
+    assert [entry["guardrail_name"] for entry in recorded] == ["earlier-guard", "credentials-raw", "credentials-raw"]
+
+
 @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
 @pytest.mark.asyncio
 async def test_escalation_step1_fails_step2_blocks(monkeypatch):

From 35d1d40a6784c222714e880c7d07729bc90018b2 Mon Sep 17 00:00:00 2001
From: yujonglee <yujong@berri.ai>
Date: Tue, 8 Sep 2026 12:07:25 -0700
Subject: [PATCH 20/27] fix(ocr): run post-call logging hooks (#40154)

---
 litellm/integrations/custom_guardrail.py      | 10 +--
 litellm/llms/custom_httpx/llm_http_handler.py | 12 ++++
 .../test_secret_detection.py                  |  2 +-
 .../integrations/test_custom_guardrail.py     | 33 ++++++++++
 .../custom_httpx/test_llm_http_handler.py     | 64 +++++++++++++++++++
 .../test_proxy_logging_hook_detection.py      | 10 +--
 6 files changed, 121 insertions(+), 10 deletions(-)

diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py
index 2d66a280663..e76d4b05bb2 100644
--- a/litellm/integrations/custom_guardrail.py
+++ b/litellm/integrations/custom_guardrail.py
@@ -773,7 +773,7 @@ class CustomGuardrail(CustomLogger):
     def uses_apply_guardrail_interface(self) -> bool:
         return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
 
-    def _deployment_pre_call_target(self) -> "CustomLogger":
+    def _deployment_hook_target(self) -> "CustomLogger":
         if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
             return self
         try:
@@ -802,7 +802,7 @@ class CustomGuardrail(CustomLogger):
 
         # CHECK IF GUARDRAIL REJECTS THE REQUEST
         if call_type == CallTypes.completion or call_type == CallTypes.acompletion:
-            target: Final = self._deployment_pre_call_target()
+            target: Final = self._deployment_hook_target()
             if target is not self:
                 kwargs["guardrail_to_apply"] = self
             result: Final = await target.async_pre_call_hook(
@@ -845,7 +845,9 @@ class CustomGuardrail(CustomLogger):
             return None
 
         # CHECK IF GUARDRAIL REJECTS THE REQUEST
-        result: Final = await self.async_post_call_success_hook(
+        target: Final = self._deployment_hook_target()
+        hook_request_data: Final = {**request_data, "guardrail_to_apply": self} if target is not self else request_data
+        result: Final = await target.async_post_call_success_hook(
             user_api_key_dict=UserAPIKeyAuth(
                 user_id=request_data.get("user_api_key_user_id"),
                 team_id=request_data.get("user_api_key_team_id"),
@@ -853,7 +855,7 @@ class CustomGuardrail(CustomLogger):
                 api_key=request_data.get("user_api_key_hash"),
                 request_route=request_data.get("user_api_key_request_route"),
             ),
-            data=request_data,
+            data=hook_request_data,
             response=response,
         )
 
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 2f561809940..8b6d3a0ceb4 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -1741,6 +1741,12 @@ class BaseLLMHTTPHandler:
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
 
+        logging_obj.post_call(
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+
         return self._transform_ocr_response(
             provider_config=provider_config,
             model=model,
@@ -1804,6 +1810,12 @@ class BaseLLMHTTPHandler:
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
 
+        logging_obj.post_call(
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+
         # Use async response transform for async operations
         return await provider_config.async_transform_ocr_response(
             model=model,
diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py
index f46df5baadf..c1c3569c0e2 100644
--- a/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py
+++ b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py
@@ -262,7 +262,7 @@ def test_proxied_traffic_stays_on_native_hooks():
     never sees ``data["prompt"]``."""
     guardrail = _guardrail()
     assert guardrail.uses_apply_guardrail_interface() is True
-    assert guardrail._deployment_pre_call_target() is guardrail
+    assert guardrail._deployment_hook_target() is guardrail
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py
index 885bd1d4d72..359635335b3 100644
--- a/tests/test_litellm/integrations/test_custom_guardrail.py
+++ b/tests/test_litellm/integrations/test_custom_guardrail.py
@@ -2610,3 +2610,36 @@ class TestCustomGuardrailPostCallSuccessDeploymentHook:
         )
 
         assert result is replacement
+
+    @pytest.mark.asyncio
+    async def test_apply_guardrail_interface_modifies_deployment_response(self):
+        from litellm.types.guardrails import GuardrailEventHooks
+        from litellm.types.utils import ModelResponse
+
+        class ReplacingGuardrail(CustomGuardrail):
+            async def apply_guardrail(
+                self,
+                inputs: GenericGuardrailAPIInputs,
+                request_data: dict[str, object],
+                input_type: Literal["request", "response"],
+                logging_obj: Optional["LiteLLMLoggingObj"] = None,
+            ) -> GenericGuardrailAPIInputs:
+                assert input_type == "response"
+                return {**inputs, "texts": ["filtered response"]}
+
+        guardrail = ReplacingGuardrail(
+            guardrail_name="test-guardrail",
+            event_hook=GuardrailEventHooks.post_call,
+        )
+        response = ModelResponse(choices=[{"message": {"role": "assistant", "content": "original response"}}])
+        request_data = {"guardrails": ["test-guardrail"]}
+
+        result = await guardrail.async_post_call_success_deployment_hook(
+            request_data=request_data,
+            response=response,
+            call_type=CallTypes.acompletion,
+        )
+
+        assert result is response
+        assert response.choices[0].message.content == "filtered response"
+        assert request_data == {"guardrails": ["test-guardrail"]}
diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
index e16855da8cb..98a5f4b2db5 100644
--- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
@@ -29,6 +29,7 @@ from litellm.llms.custom_httpx.llm_http_handler import (
     _rust_responses_websocket_enabled,
 )
 from litellm.llms.azure.videos.transformation import AzureVideoConfig
+from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
 from litellm.llms.openai.videos.transformation import OpenAIVideoConfig
 from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.types.router import GenericLiteLLMParams
@@ -37,6 +38,69 @@ from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, Trans
 _ACTIVE_KEY = "_code_interpreter_interception_active"
 _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
 
+OCR_RESPONSE = {
+    "pages": [{"index": 0, "markdown": "OCR output", "images": []}],
+    "model": "mistral-ocr-latest",
+    "usage_info": {"pages_processed": 1},
+}
+
+
+def _ocr_sync_client() -> HTTPHandler:
+    client = HTTPHandler()
+    client.client = httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(200, json=OCR_RESPONSE)))
+    return client
+
+
+def _ocr_async_client() -> AsyncHTTPHandler:
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(
+        transport=httpx.MockTransport(lambda _request: httpx.Response(200, json=OCR_RESPONSE))
+    )
+    return client
+
+
+def test_ocr_calls_post_call_with_raw_provider_response():
+    logging_obj = Mock()
+
+    response = BaseLLMHTTPHandler().ocr(
+        model="mistral-ocr-latest",
+        document={"type": "document_url", "document_url": "https://example.com/document.pdf"},
+        optional_params={},
+        timeout=5,
+        logging_obj=logging_obj,
+        api_key="test-key",
+        api_base="https://api.mistral.ai/v1/ocr",
+        custom_llm_provider="mistral",
+        client=_ocr_sync_client(),
+        provider_config=MistralOCRConfig(),
+    )
+
+    assert response.pages[0].markdown == "OCR output"
+    logging_obj.post_call.assert_called_once()
+    assert json.loads(logging_obj.post_call.call_args.kwargs["original_response"]) == OCR_RESPONSE
+
+
+@pytest.mark.asyncio
+async def test_async_ocr_calls_post_call_with_raw_provider_response():
+    logging_obj = Mock()
+
+    response = await BaseLLMHTTPHandler().async_ocr(
+        model="mistral-ocr-latest",
+        document={"type": "document_url", "document_url": "https://example.com/document.pdf"},
+        optional_params={},
+        timeout=5,
+        logging_obj=logging_obj,
+        api_key="test-key",
+        api_base="https://api.mistral.ai/v1/ocr",
+        custom_llm_provider="mistral",
+        client=_ocr_async_client(),
+        provider_config=MistralOCRConfig(),
+    )
+
+    assert response.pages[0].markdown == "OCR output"
+    logging_obj.post_call.assert_called_once()
+    assert json.loads(logging_obj.post_call.call_args.kwargs["original_response"]) == OCR_RESPONSE
+
 
 def test_prepare_fake_stream_request():
     # Initialize the BaseLLMHTTPHandler
diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py
index 9f1321aec2c..22930a26974 100644
--- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py
+++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py
@@ -637,14 +637,14 @@ def test_callback_capabilities_excludes_opted_out_guardrail_from_iterator_overri
     assert [cb for cb, _ in caps.iterator_overrides if cb is opted_out] == []
 
 
-def test_deployment_pre_call_target_stays_native_when_opted_out():
+def test_deployment_hook_target_stays_native_when_opted_out():
     """Model-level guardrails resolve their target here rather than through ProxyLogging."""
-    assert _KeepsNativeHooks()._deployment_pre_call_target() is not None
+    assert _KeepsNativeHooks()._deployment_hook_target() is not None
     opted_out = _KeepsNativeHooks()
-    assert opted_out._deployment_pre_call_target() is opted_out
-    assert _AppliesGuardrail()._deployment_pre_call_target() is not None
+    assert opted_out._deployment_hook_target() is opted_out
+    assert _AppliesGuardrail()._deployment_hook_target() is not None
     routed = _AppliesGuardrail()
-    assert routed._deployment_pre_call_target() is not routed
+    assert routed._deployment_hook_target() is not routed
 
 
 @pytest.mark.asyncio

From 01c68c199bc0afddf5a4d806cb03eca5e7745295 Mon Sep 17 00:00:00 2001
From: yujonglee <yujong@berri.ai>
Date: Tue, 8 Sep 2026 12:18:09 -0700
Subject: [PATCH 21/27] fix(guardrails): allow framework-supported logging-only
 mode (#40267)

---
 litellm/integrations/custom_guardrail.py      |  10 +-
 .../integrations/test_custom_guardrail.py     | 100 +++++++++++++++++-
 2 files changed, 107 insertions(+), 3 deletions(-)

diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py
index e76d4b05bb2..37d6a7e793d 100644
--- a/litellm/integrations/custom_guardrail.py
+++ b/litellm/integrations/custom_guardrail.py
@@ -601,6 +601,12 @@ class CustomGuardrail(CustomLogger):
         event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None,
         supported_event_hooks: list[GuardrailEventHooks],
     ) -> None:
+        allowed_hooks: Final = frozenset(supported_event_hooks) | (
+            frozenset((GuardrailEventHooks.logging_only,))
+            if self.uses_apply_guardrail_interface() and not self.use_native_lifecycle_hooks
+            else frozenset()
+        )
+
         def _validate_event_hook_list_is_in_supported_event_hooks(
             event_hook: list[GuardrailEventHooks] | list[str],
             supported_event_hooks: list[GuardrailEventHooks],
@@ -608,7 +614,7 @@ class CustomGuardrail(CustomLogger):
             for hook in event_hook:
                 if isinstance(hook, str):
                     hook = GuardrailEventHooks(hook)
-                if hook not in supported_event_hooks:
+                if hook not in allowed_hooks:
                     raise ValueError(f"Event hook {hook} is not in the supported event hooks {supported_event_hooks}")
 
         if event_hook is None:
@@ -629,7 +635,7 @@ class CustomGuardrail(CustomLogger):
                 default_list = event_hook.default if isinstance(event_hook.default, list) else [event_hook.default]
                 _validate_event_hook_list_is_in_supported_event_hooks(default_list, supported_event_hooks)
         elif isinstance(event_hook, GuardrailEventHooks):
-            if event_hook not in supported_event_hooks:
+            if event_hook not in allowed_hooks:
                 raise ValueError(f"Event hook {event_hook} is not in the supported event hooks {supported_event_hooks}")
 
     @staticmethod
diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py
index 359635335b3..cd8d609cf71 100644
--- a/tests/test_litellm/integrations/test_custom_guardrail.py
+++ b/tests/test_litellm/integrations/test_custom_guardrail.py
@@ -1,5 +1,5 @@
 import asyncio
-from typing import TYPE_CHECKING, Literal, Optional
+from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional
 from unittest.mock import AsyncMock
 
 import pytest
@@ -10,6 +10,7 @@ from litellm.integrations.custom_guardrail import (
     log_guardrail_information,
 )
 from litellm.proxy._types import CallTypes, UserAPIKeyAuth
+from litellm.types.guardrails import GuardrailEventHooks, Mode
 from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailTracingDetail
 
 if TYPE_CHECKING:
@@ -2378,11 +2379,108 @@ def _logged_call(messages: list | str) -> tuple[dict, object]:
     return kwargs, response
 
 
+class _NativeApplyGuardrail(_InheritedApplyGuardrail):
+    use_native_lifecycle_hooks: ClassVar[bool] = True
+
+
+@pytest.mark.parametrize("guardrail_type", (CustomGuardrail, _NativeApplyGuardrail, _InheritedApplyGuardrail))
+@pytest.mark.parametrize(
+    "event_hook",
+    (
+        GuardrailEventHooks.logging_only,
+        "logging_only",
+        [GuardrailEventHooks.pre_call, GuardrailEventHooks.logging_only],
+        ["pre_call", "logging_only"],
+        Mode(tags={"audit": "logging_only"}, default="pre_call"),
+        Mode(tags={"audit": ["pre_call", "logging_only"]}),
+        Mode(tags={"enforce": "pre_call"}, default="logging_only"),
+        Mode(tags={}, default=["pre_call", "logging_only"]),
+    ),
+)
+def test_logging_only_requires_framework_support_or_explicit_declaration(
+    guardrail_type: type[CustomGuardrail],
+    event_hook: GuardrailEventHooks | str | list[GuardrailEventHooks] | list[str] | Mode,
+) -> None:
+    supported: Final = [GuardrailEventHooks.pre_call]
+    if guardrail_type is _InheritedApplyGuardrail:
+        guardrail: Final = guardrail_type(event_hook=event_hook, supported_event_hooks=supported)
+        assert guardrail.event_hook == event_hook
+        assert supported == [GuardrailEventHooks.pre_call]
+    else:
+        with pytest.raises(ValueError, match=r"logging_only.*not in the supported event hooks"):
+            guardrail_type(event_hook=event_hook, supported_event_hooks=supported)
+
+    explicitly_supported: Final = guardrail_type(
+        event_hook=event_hook,
+        supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.logging_only],
+    )
+    assert explicitly_supported.event_hook == event_hook
+
+
+@pytest.mark.parametrize(
+    "event_hook",
+    (
+        GuardrailEventHooks.post_call,
+        "post_call",
+        [GuardrailEventHooks.logging_only, GuardrailEventHooks.post_call],
+        ["logging_only", "post_call"],
+        Mode(tags={"enforce": "post_call"}, default="logging_only"),
+        Mode(tags={"enforce": ["logging_only", "post_call"]}),
+        Mode(tags={"audit": "logging_only"}, default="post_call"),
+        Mode(tags={}, default=["logging_only", "post_call"]),
+    ),
+)
+def test_framework_logging_only_does_not_allow_other_unsupported_modes(
+    event_hook: GuardrailEventHooks | str | list[GuardrailEventHooks] | list[str] | Mode,
+) -> None:
+    with pytest.raises(ValueError, match=r"post_call.*not in the supported event hooks"):
+        _InheritedApplyGuardrail(event_hook=event_hook, supported_event_hooks=[GuardrailEventHooks.pre_call])
+
+
 class TestLoggingOnlyApplyGuardrail:
     """LIT-4876 regression: a guardrail in mode logging_only that implements only
     apply_guardrail must still run against the logged request and response and
     record guardrail_information, instead of inheriting the CustomLogger no-op."""
 
+    @pytest.mark.parametrize(
+        "event_hook",
+        (
+            GuardrailEventHooks.logging_only,
+            "logging_only",
+            [GuardrailEventHooks.pre_call, GuardrailEventHooks.logging_only],
+            ["pre_call", "logging_only"],
+            Mode(tags={"audit": "logging_only"}, default="pre_call"),
+            Mode(tags={"audit": ["pre_call", "logging_only"]}),
+            Mode(tags={"enforce": "pre_call"}, default="logging_only"),
+            Mode(tags={}, default=["pre_call", "logging_only"]),
+        ),
+    )
+    @pytest.mark.asyncio
+    async def test_content_filter_accepts_logging_only_and_records_detection(
+        self, event_hook: GuardrailEventHooks | str | list[GuardrailEventHooks] | list[str] | Mode
+    ) -> None:
+        from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import (
+            ContentFilterGuardrail,
+        )
+        from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks
+
+        guardrail: Final = ContentFilterGuardrail(
+            guardrail_name="content-review",
+            event_hook=event_hook,
+            default_on=True,
+            blocked_words=[BlockedWord(keyword="hello", action=ContentFilterAction.BLOCK)],
+        )
+        kwargs, response = _logged_call([{"role": "user", "content": "hello there"}])
+
+        out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
+
+        assert out_response is response
+        assert out_kwargs["messages"] == kwargs["messages"]
+        assert (
+            out_kwargs["standard_logging_object"]["guardrail_information"][0]["guardrail_status"]
+            == "guardrail_intervened"
+        )
+
     @pytest.mark.asyncio
     async def test_runs_apply_guardrail_observe_only_and_records_verdict(self):
         guardrail = _ApplyOnlyObserver()

From d36e032241490a5a75063425a99da4ccdc85548a Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:00:14 -0700
Subject: [PATCH 22/27] fix(proxy): initialize string success/failure callbacks
 at startup after config load (#38226)

* fix(proxy): eagerly initialize string callbacks

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): materialize string callbacks after load_config so later litellm_settings keys are applied

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): drop casts when snapshotting string callbacks so LIT006 stays at base

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yucheng <yucheng@berri.ai>
---
 litellm/proxy/utils.py                        | 11 +++++
 tests/test_litellm/proxy/test_proxy_server.py | 41 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index ddf31cb1d8a..75d7972c621 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -177,6 +177,9 @@ from litellm.types.mcp import (
 )
 from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionResult
 from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams
+from litellm.utils import (
+    _add_custom_logger_callback_to_specific_event,  # pyright: ignore[reportPrivateUsage]  # only string-to-logger helper
+)
 
 if TYPE_CHECKING:
     from mcp.types import CallToolResult
@@ -857,6 +860,14 @@ class ProxyLogging:
                 litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
                 litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
 
+        # Runs after load_config applied every litellm_settings key: logger __init__s read e.g. s3_callback_params
+        success_callbacks: Final = tuple(cb for cb in litellm.success_callback if isinstance(cb, str))
+        failure_callbacks: Final = tuple(cb for cb in litellm.failure_callback if isinstance(cb, str))
+        for callback in success_callbacks:
+            _add_custom_logger_callback_to_specific_event(callback, "success")
+        for callback in failure_callbacks:
+            _add_custom_logger_callback_to_specific_event(callback, "failure")
+
     async def update_request_status(self, litellm_call_id: str, status: Literal["success", "fail"]):
         # only use this if slack alerting is being used
         if self.alerting is None:
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 937e4f15741..6a901c5e7de 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -3166,6 +3166,47 @@ async def test_custom_ui_sso_sign_in_handler_config_loading():
         os.unlink(config_file_path)
 
 
+@pytest.mark.asyncio
+async def test_startup_initializes_string_callbacks_after_all_litellm_settings_load(tmp_path, monkeypatch):
+    from litellm.integrations.s3_v2 import S3Logger
+    from litellm.litellm_core_utils import litellm_logging
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.proxy.utils import ProxyLogging
+
+    config_file = tmp_path / "config.yaml"
+    config_file.write_text(
+        "model_list: []\n"
+        "litellm_settings:\n"
+        "  success_callback:\n"
+        "    - s3_v2\n"
+        "  failure_callback:\n"
+        "    - s3_v2\n"
+        "  s3_callback_params:\n"
+        "    s3_bucket_name: ordering-regression-bucket\n"
+        "    s3_region_name: us-west-2\n"
+    )
+
+    monkeypatch.setattr(litellm, "success_callback", [])
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "failure_callback", [])
+    monkeypatch.setattr(litellm, "_async_failure_callback", [])
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm, "s3_callback_params", None)
+    monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
+
+    await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file))
+    ProxyLogging(user_api_key_cache=MagicMock())._init_litellm_callbacks(llm_router=None)
+
+    success_loggers = [cb for cb in litellm._async_success_callback if isinstance(cb, S3Logger)]
+    failure_loggers = [cb for cb in litellm._async_failure_callback if isinstance(cb, S3Logger)]
+    assert len(success_loggers) == 1
+    assert len(failure_loggers) == 1
+    assert success_loggers[0].s3_bucket_name == "ordering-regression-bucket"
+    assert success_loggers[0].s3_region_name == "us-west-2"
+    assert "s3_v2" not in litellm.success_callback
+    assert "s3_v2" not in litellm.failure_callback
+
+
 @pytest.mark.asyncio
 async def test_load_config_max_budget_env_var_coerced_to_float(tmp_path, monkeypatch):
     """

From 6a425a5cc59ce67f3860ac04f752cf61340e1028 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:08:26 -0700
Subject: [PATCH 23/27] fix(responses): record spend for native Responses API
 WebSocket sessions (#38856)

* fix(responses): record spend for native Responses API WebSocket sessions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): bill usage from response.incomplete WebSocket turns

A turn cut short by max_output_tokens ends in response.incomplete, which
OpenAI bills but the processor only read response.completed, so those
sessions still logged zero spend

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(responses): hoist websocket usage test imports to module scope

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): price websocket sessions through the standard cost path

The realtime completion_cost branch skips cost_discount_config and cost_margin_config, so a native
Responses WebSocket session was priced differently from the same usage over HTTP /v1/responses.
Drop the explicit widening so the LiteLLMRealtimeStreamLoggingObject built by
normalize_logging_result flows through the generic usage path, and pin WS == HTTP cost under a
50% provider discount in the regression test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: rerun proxy-infra after flaky test_check_migration process tree test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yucheng <yucheng@berri.ai>
---
 litellm/cost_calculator.py                    | 40 ++++++++
 litellm/litellm_core_utils/litellm_logging.py | 12 +++
 litellm/types/utils.py                        |  1 +
 .../test_litellm_logging.py                   | 92 ++++++++++++++++++-
 4 files changed, 144 insertions(+), 1 deletion(-)

diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 9a9d2ceda03..c9c7df4d75e 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -2414,6 +2414,46 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
         )
 
 
+_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
+
+
+class _ResponsesWsEventResponse(BaseModel):
+    usage: Mapping[str, object] | None = None
+
+
+class _ResponsesWsEvent(BaseModel):
+    type: str = ""
+    response: _ResponsesWsEventResponse | None = None
+
+
+class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
+    @staticmethod
+    def collect_usage_from_responses_ws_results(
+        results: Sequence[Mapping[str, object]],
+    ) -> tuple[Usage, ...]:
+        events: Final = tuple(_ResponsesWsEvent.model_validate(result) for result in results)
+        return tuple(
+            ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(  # pyright: ignore[reportPrivateUsage]  # same shared transform the realtime processor uses
+                event.response.usage
+            )
+            for event in events
+            if event.type in _RESPONSES_WS_BILLABLE_EVENT_TYPES
+            and event.response is not None
+            and event.response.usage is not None
+        )
+
+    @staticmethod
+    def collect_and_combine_usage_from_responses_ws_results(
+        results: Sequence[Mapping[str, object]],
+    ) -> Usage:
+        collected_usage_objects: Final = ResponsesWebSocketTokenUsageProcessor.collect_usage_from_responses_ws_results(
+            results
+        )
+        return ResponsesWebSocketTokenUsageProcessor.combine_usage_objects(
+            list(collected_usage_objects)  # mutable-ok: combine_usage_objects requires a list parameter
+        )
+
+
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
 
 
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index ca2cca5360f..e3c94866985 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -48,6 +48,7 @@ from litellm.constants import (
 )
 from litellm.cost_calculator import (
     RealtimeAPITokenUsageProcessor,
+    ResponsesWebSocketTokenUsageProcessor,
     _select_model_name_for_cost_calc,
 )
 from litellm.exceptions import (
@@ -2028,6 +2029,17 @@ class Logging(LiteLLMLoggingBaseClass):
                 results=result,
             )
 
+        elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list):  # pyright: ignore[reportUnknownMemberType]  # Logging.call_type is untyped
+            combined_ws_usage: Final = (
+                ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
+                    results=result  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                )
+            )
+            logging_result = LiteLLMRealtimeStreamLoggingObject(
+                usage=combined_ws_usage,
+                results=result,  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+            )
+
         elif (
             self.call_type == CallTypes.llm_passthrough_route.value
             or self.call_type == CallTypes.allm_passthrough_route.value
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 9b5fb08a45f..40a6d77482f 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -580,6 +580,7 @@ CallTypesLiteral = Literal[
     "search",
     "asearch",
     "_arealtime",
+    "_aresponses_websocket",
     "create_batch",
     "acreate_batch",
     "create_file",
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index 0fdca755685..b6366bc803e 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -22,7 +22,13 @@ from litellm.litellm_core_utils.litellm_logging import (
     _get_status_fields,
     set_callbacks,
 )
-from litellm.types.utils import ModelResponse, TextCompletionResponse
+from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
+from litellm.types.utils import (
+    CallTypes,
+    LiteLLMRealtimeStreamLoggingObject,
+    ModelResponse,
+    TextCompletionResponse,
+)
 
 
 @pytest.fixture
@@ -6393,6 +6399,90 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o
     assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == ""
 
 
+def _responses_ws_logging_obj() -> LitellmLogging:
+    return LitellmLogging(
+        model="gpt-4o",
+        messages=[],
+        stream=False,
+        call_type=CallTypes.aresponses_websocket.value,
+        start_time=time.time(),
+        litellm_call_id="responses-ws-usage-test",
+        function_id="responses-ws-usage-test",
+    )
+
+
+def test_normalize_logging_result_extracts_usage_for_responses_websocket(monkeypatch):
+    """LIT-6512: native /v1/responses WebSocket sessions logged $0 spend because the usage
+    carried by stored response.completed events was never extracted. The session must cost
+    exactly what the same usage costs over HTTP /v1/responses, discounts included."""
+    monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.5})
+    logging_obj = _responses_ws_logging_obj()
+    events = [
+        {"type": "response.created", "response": {}},
+        {
+            "type": "response.completed",
+            "response": {"usage": {"input_tokens": 100, "output_tokens": 40, "total_tokens": 140}},
+        },
+        {
+            "type": "response.completed",
+            "response": {"usage": {"input_tokens": 60, "output_tokens": 10, "total_tokens": 70}},
+        },
+    ]
+
+    normalized = logging_obj.normalize_logging_result(result=events)
+
+    assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject)
+    assert normalized.usage.prompt_tokens == 160
+    assert normalized.usage.completion_tokens == 50
+
+    ws_cost = litellm.completion_cost(
+        completion_response=normalized,
+        model="gpt-4o",
+        call_type=CallTypes.aresponses_websocket.value,
+        custom_llm_provider="openai",
+    )
+    http_cost = litellm.completion_cost(
+        completion_response=ResponsesAPIResponse(
+            id="resp-6512",
+            created_at=1700000000,
+            output=[],
+            usage=ResponseAPIUsage(input_tokens=160, output_tokens=50, total_tokens=210),
+        ),
+        model="gpt-4o",
+        call_type=CallTypes.aresponses.value,
+        custom_llm_provider="openai",
+    )
+    assert ws_cost > 0
+    assert ws_cost == http_cost
+
+
+def test_normalize_logging_result_bills_incomplete_responses_websocket_turns():
+    """LIT-6512: a turn cut short by max_output_tokens ends in response.incomplete, which
+    OpenAI bills, so its usage counts toward the session like a completed turn."""
+    events = [
+        {
+            "type": "response.created",
+            "response": {"usage": {"input_tokens": 999, "output_tokens": 999, "total_tokens": 1998}},
+        },
+        {
+            "type": "response.incomplete",
+            "response": {"usage": {"input_tokens": 15, "output_tokens": 16, "total_tokens": 31}},
+        },
+        {
+            "type": "response.completed",
+            "response": {"usage": {"input_tokens": 40, "output_tokens": 4, "total_tokens": 44}},
+        },
+        {"type": "response.failed", "response": {"usage": None}},
+    ]
+
+    normalized = _responses_ws_logging_obj().normalize_logging_result(result=events)
+
+    assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject)
+    assert normalized.usage.prompt_tokens == 55
+    assert normalized.usage.completion_tokens == 20
+    assert normalized.usage.total_tokens == 75
+
+
 def test_get_standard_logging_object_payload_reads_overhead_from_logging_obj_for_dict_results(logging_obj):
     """LIT-5466: /v1/messages returns a plain dict with no _hidden_params, so the overhead
     recorded on the logging object must reach hidden_params.litellm_overhead_time_ms (SpendLogs)."""

From 9d164fa34150ac7dc0551d9efec16cdb27848c5c Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 8 Sep 2026 13:24:15 -0700
Subject: [PATCH 24/27] bump: litellm-proxy-extras 0.4.94 -> 0.4.95, litellm
 1.101.0 -> 1.102.0

---
 litellm-proxy-extras/pyproject.toml | 4 ++--
 pyproject.toml                      | 6 +++---
 uv.lock                             | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index 82d31fec373..91b4e4a7ba1 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.94"
+version = "0.4.95"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.94"
+version = "0.4.95"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",
diff --git a/pyproject.toml b/pyproject.toml
index af35c77d259..706b14b8a04 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.101.0"
+version = "1.102.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -67,7 +67,7 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.94",
+    "litellm-proxy-extras==0.4.95",
     "litellm-enterprise==0.1.65",
     "RestrictedPython>=8.5,<9.0",
     "rich>=13.9.4,<14.0",
@@ -328,7 +328,7 @@ members = ["enterprise", "litellm-proxy-extras"]
 profile = "black"
 
 [tool.commitizen]
-version = "1.101.0"
+version = "1.102.0"
 version_files = [
     "pyproject.toml:^version",
 ]
diff --git a/uv.lock b/uv.lock
index 89205cd9527..12dc2c69d23 100644
--- a/uv.lock
+++ b/uv.lock
@@ -10,7 +10,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-02T16:58:34.594994Z"
+exclude-newer = "2026-09-05T20:24:07.535116Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -4358,7 +4358,7 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.101.0"
+version = "1.102.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },
@@ -4776,7 +4776,7 @@ source = { editable = "enterprise" }
 
 [[package]]
 name = "litellm-proxy-extras"
-version = "0.4.94"
+version = "0.4.95"
 source = { editable = "litellm-proxy-extras" }
 
 [[package]]

From 4a3a78c256bab366597a76d44930b323bce6335b Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Tue, 8 Sep 2026 13:28:57 -0700
Subject: [PATCH 25/27] feat(complexity_router): rebalance heuristic weights in
 the dashboard and grade custom dimensions by match count (#40205)

The Advanced scoring editor now lists built-in and custom dimensions together. Editing any weight holds it and rescales the others proportionally so the vector totals 1.00, and Save stores those explicit values. The backend scores exactly what is stored, with no runtime normalization, so routers nobody edits keep their weights.

CustomDimension gains an opt-in scoring_mode. match_count scores 0, 0.5 or 1 by distinct matcher hits; the default stays binary. The tuning fingerprint omits a binary scoring_mode, so routers written before this change keep their recorded baseline and the upgrade does not consume the free heuristic-v1 tuning slot.
---
 .../complexity_router/README.md               |  12 +-
 .../complexity_router/complexity_router.py    |  38 ++++-
 .../complexity_router/config.py               |  13 +-
 .../auto_router_tuning_baseline.py            |  12 +-
 .../router_strategy/test_complexity_router.py | 148 +++++++++++++-----
 .../test_auto_router_tuning_baseline.py       |  73 ++++++++-
 .../add_model/ClassificationMethodConfig.tsx  |   5 +-
 .../add_model/ComplexityRouterConfig.test.tsx |   6 +-
 .../add_model/ComplexityRouterConfig.tsx      |   6 +
 .../add_model/CustomDimensionRows.tsx         | 136 ++++++++++++++++
 ...euristicScoringConfig.integration.test.tsx |  90 +++++++++++
 .../add_model/HeuristicScoringConfig.test.tsx |  14 +-
 .../add_model/HeuristicScoringConfig.tsx      |  85 ++++++++--
 .../add_model/add_auto_router_tab.tsx         |   4 +
 .../build_complexity_router_config.test.ts    |  47 +++++-
 .../build_complexity_router_config.ts         |  17 +-
 .../add_model/custom_dimensions.test.ts       |  41 +++++
 .../components/add_model/custom_dimensions.ts |  54 +++++++
 .../add_model/heuristic_scoring_knobs.test.ts | 119 ++++++++++++++
 .../add_model/heuristic_scoring_knobs.ts      |  97 +++++++++++-
 .../src/components/add_model/tier_rows.ts     |   1 +
 ...d_updated_complexity_router_config.test.ts |  49 ++++++
 .../edit_auto_router_modal.tsx                |  17 +-
 .../src/lib/autorouter_presets.test.ts        |  28 ++++
 .../src/lib/autorouter_presets.ts             |  12 ++
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   9 +-
 26 files changed, 1047 insertions(+), 86 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts
 create mode 100644 ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts

diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md
index 88ed374dd3f..d605b43e42a 100644
--- a/litellm/router_strategy/complexity_router/README.md
+++ b/litellm/router_strategy/complexity_router/README.md
@@ -207,17 +207,27 @@ custom_dimensions:
   - name: sqlMigration
     weight: 0.7
     patterns: ['\b(create|alter|drop)\s{1,4}table\b']
+  - name: dataPipeline
+    weight: 0.4
+    scoring_mode: match_count
+    keywords: [airflow, dbt, snowflake]
 ```
 
 Each dimension contributes its weight once when any matcher hits the current ask. Repeated matches do not increase it. The built-in score and tier boundaries are unchanged, and the total score is not renormalized. Keywords use the existing case-insensitive word-boundary and CJK rules. Regexes search the first 2048 characters case-insensitively and compile during configuration validation and router initialization, never per request
 
+`scoring_mode` is optional and defaults to `binary`, the behavior above. `match_count` grades the dimension by how many distinct matchers hit: none scores 0 and emits no signal, one scores half the weight, two or more score the full weight. Repeated occurrences of one matcher never raise the count, keywords are distinct case-insensitively, patterns are distinct by source, and a keyword and a pattern are always distinct from each other. Matching stops as soon as the selected mode's maximum is reached, so a binary dimension still stops at its first hit. Existing configurations without the field keep binary scoring and the same tuning fingerprint, so the field only counts as a tuning change when set to `match_count`
+
+### Weights through the API versus the dashboard
+
+The API and YAML store exactly the weights written. A `dimension_weights` map and inline custom weights are read literally, missing recognized built-in names score zero, and nothing renormalizes the vector, so a total other than 1 is legal and scores accordingly. The dashboard's heuristic scoring editor is the one place that rebalances: editing one weight there holds it and redistributes the remainder across the other active dimensions in the draft, then Save sends the resulting explicit values, which the backend stores and scores as written. Opening a router, applying a preset, editing matchers, changing `scoring_mode`, or saving unrelated fields never normalizes existing weights
+
 Only `heuristic`, `heuristic_first` and `hybrid` accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one
 
 Patterns are checked at configuration time against a grammar whose worst case stays a few milliseconds on 2048 characters. Every quantifier needs an explicit upper bound of at most 64 and must repeat a single character or character class, so `\s{1,4}` is accepted while `\s+`, `(a|aa){0,12}` and `(?:ab){0,64}` are refused. Backreferences, lookarounds, atomic groups and possessive quantifiers are refused as well. Each pattern is then costed: alternation branches and repeat lengths multiply the ways the engine can retry, and every later piece of the pattern is charged once per path that can reach it, so `a?a?a?a?a?a?a?a?` followed by a long fixed tail is refused even though each quantifier is small. The budget is 2048 work units per pattern and 8192 across the router. An invalid or over-budget pattern fails the write with a message naming the pattern and the rule it broke
 
 Limits are 16 dimensions, 32 combined keywords/patterns per dimension, 256 characters per matcher and 4096 matcher characters per dimension. Matching runs inline on the request path with no timeout and no worker thread, because the grammar is what bounds the cost. These are routing hints, not security enforcement rules
 
-The existing heuristic-v1 tuning quota covers custom dimensions and their weights: one changed router without an auto-router license, unlimited with the entitlement. Omitting `custom_dimensions` preserves existing scoring. Routing decisions and spend logs include signals such as `custom (sqlMigration)` without recording the configured pattern or matched text. The field is configured through YAML or the model API; this change adds no dashboard editor
+The existing heuristic-v1 tuning quota covers custom dimensions, their weights and their scoring mode: one changed router without an auto-router license, unlimited with the entitlement. Omitting `custom_dimensions` preserves existing scoring. Routing decisions and spend logs include signals such as `custom (sqlMigration)` without recording the configured pattern or matched text. The field is configured through YAML, the model API, or the dashboard's heuristic scoring editor
 
 ## Usage
 
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index c8644f52c57..ae1f82fc40b 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -20,7 +20,7 @@ import random
 import re
 import time
 from collections.abc import Callable, Iterator, Mapping, Sequence
-from itertools import accumulate, islice, takewhile
+from itertools import accumulate, chain, islice, takewhile
 from threading import Lock
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
@@ -82,6 +82,7 @@ from .config import (
     ClassificationRubric,
     ComplexityRouterConfig,
     ComplexityTier,
+    CustomDimension,
     TierDefinition,
 )
 from .stall_detector import detect_stalled_task
@@ -879,6 +880,15 @@ class DimensionScore:
         self.signal = signal
 
 
+class _CustomDimensionMatchers(NamedTuple):
+    """One custom dimension's distinct matchers and the number of hits that saturates its score."""
+
+    dimension: CustomDimension
+    keywords: tuple[str, ...]
+    patterns: tuple[re.Pattern[str], ...]
+    saturation: int
+
+
 class KeywordOverride(NamedTuple):
     """A keyword_tier_rules match: the winning tier and, on the lexical path, the keyword that fired."""
 
@@ -1121,7 +1131,12 @@ class ComplexityRouter(CustomLogger):
         )
         self.simple_keywords = self.config.simple_keywords or DEFAULT_SIMPLE_KEYWORDS
         self._custom_dimensions = tuple(
-            (dimension, tuple(re.compile(pattern, re.IGNORECASE) for pattern in dimension.patterns))
+            _CustomDimensionMatchers(
+                dimension,
+                tuple(dict.fromkeys(keyword.lower() for keyword in dimension.keywords)),
+                tuple(re.compile(pattern, re.IGNORECASE) for pattern in dict.fromkeys(dimension.patterns)),
+                2 if dimension.scoring_mode == "match_count" else 1,
+            )
             for dimension in self.config.custom_dimensions
         )
         if self.config.has_custom_tiers:
@@ -1325,15 +1340,26 @@ class ComplexityRouter(CustomLogger):
         score: Final = score_high if match_count >= high_threshold else score_low
         return DimensionScore(name, score, f"{signal_label} ({detail})"), match_count
 
+    def _count_custom_hits(self, matchers: _CustomDimensionMatchers, user_text: str, scanned: str) -> int:
+        hits: Final = chain(
+            (self._keyword_matches(user_text, keyword) for keyword in matchers.keywords),
+            (pattern.search(scanned) is not None for pattern in matchers.patterns),
+        )
+        return sum(islice((1 for hit in hits if hit), matchers.saturation))
+
     def _score_custom_dimensions(self, prompt: str, user_text: str) -> tuple[tuple[DimensionScore, float], ...]:
         if not self._custom_dimensions:
             return ()
         scanned: Final = prompt[:CUSTOM_PATTERN_SCAN_CHARS]
         return tuple(
-            (DimensionScore(dimension.name, 1.0, f"custom ({dimension.name})"), dimension.weight)
-            for dimension, patterns in self._custom_dimensions
-            if any(self._keyword_matches(user_text, keyword) for keyword in dimension.keywords)
-            or any(pattern.search(scanned) is not None for pattern in patterns)
+            (
+                DimensionScore(
+                    matchers.dimension.name, hits / matchers.saturation, f"custom ({matchers.dimension.name})"
+                ),
+                matchers.dimension.weight,
+            )
+            for matchers in self._custom_dimensions
+            if (hits := self._count_custom_hits(matchers, user_text, scanned))
         )
 
     def _score_multi_step(self, text: str) -> DimensionScore:
diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py
index 3ec9f9b5394..37375d9727d 100644
--- a/litellm/router_strategy/complexity_router/config.py
+++ b/litellm/router_strategy/complexity_router/config.py
@@ -667,6 +667,14 @@ class CustomDimension(BaseModel):
     weight: float = Field(gt=0, le=1, allow_inf_nan=False)
     keywords: tuple[Annotated[str, Field(min_length=1, max_length=256)], ...] = Field(default=(), max_length=32)
     patterns: tuple[Annotated[str, Field(min_length=1, max_length=256)], ...] = Field(default=(), max_length=32)
+    scoring_mode: Literal["binary", "match_count"] = Field(
+        default="binary",
+        description=(
+            "'binary' scores 1 when any matcher hits. 'match_count' scores 0.5 when one distinct matcher hits and 1 "
+            "when two or more do; repeated occurrences of one matcher never raise it. Keywords are distinct "
+            "case-insensitively, patterns by source, and a keyword and a pattern are always distinct from each other."
+        ),
+    )
 
     @model_validator(mode="after")
     def _validate_matchers(self) -> "CustomDimension":
@@ -794,8 +802,9 @@ class ComplexityRouterConfig(BaseModel):
         default=(),
         max_length=16,
         description=(
-            "Named binary dimensions added to the heuristic-v1 score. Each contributes its inline weight once "
-            "when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters. "
+            "Named dimensions added to the heuristic-v1 score. Each contributes its inline weight once "
+            "when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters; "
+            "scoring_mode 'match_count' instead grades half weight for one distinct matcher and full for two or more. "
             "Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, "
             "backreferences and lookarounds are rejected. Conservative work limits include alternation paths, "
             "repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. "
diff --git a/litellm/router_utils/auto_router_tuning_baseline.py b/litellm/router_utils/auto_router_tuning_baseline.py
index 74f7b82389a..9699ab886b9 100644
--- a/litellm/router_utils/auto_router_tuning_baseline.py
+++ b/litellm/router_utils/auto_router_tuning_baseline.py
@@ -52,7 +52,17 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None:
     supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | (
         frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset()
     )
-    payload: Final = validated.model_dump(mode="json", include=supplied)
+    payload: Final = validated.model_dump(
+        mode="json",
+        include=supplied,
+        exclude={
+            "custom_dimensions": {
+                index: {"scoring_mode"}
+                for index, dimension in enumerate(validated.custom_dimensions)
+                if dimension.scoring_mode == "binary"
+            }
+        },
+    )
     return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
 
 
diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py
index 5b1d8562abd..b0f2a65bb9b 100644
--- a/tests/test_litellm/router_strategy/test_complexity_router.py
+++ b/tests/test_litellm/router_strategy/test_complexity_router.py
@@ -826,6 +826,8 @@ class TestCustomDimensions:
             pytest.param({"keywords": ["x"] * 32, "patterns": ["y"]}, {}, id="combined-matcher-count"),
             pytest.param({"keywords": ["x" * 256] * 17}, {}, id="matcher-character-budget"),
             pytest.param({"unknown": True}, {}, id="extra-field"),
+            pytest.param({"scoring_mode": "graded"}, {}, id="unknown-scoring-mode"),
+            pytest.param({"scoring_mode": None}, {}, id="null-scoring-mode"),
         ],
     )
     def test_custom_dimension_invalid_configuration_rejected(
@@ -878,43 +880,125 @@ class TestCustomDimensions:
             )
 
     @pytest.mark.asyncio
-    @pytest.mark.parametrize("current_ask", ("Hello!", "orbitmesh"))
+    @pytest.mark.parametrize("scoring_mode", ("binary", "match_count"))
+    @pytest.mark.parametrize("current_ask", ("Hello!", "orbitmesh", "orbitmesh fluxgate"))
     async def test_custom_dimensions_public_hook_scores_only_current_ask(
-        self, mock_router_instance: MagicMock, current_ask: str
+        self, mock_router_instance: MagicMock, current_ask: str, scoring_mode: str
     ) -> None:
         router: Final = ComplexityRouter(
             "test-router",
             mock_router_instance,
             {
                 "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"},
-                "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}],
+                "dimension_weights": {},
+                "custom_dimensions": [
+                    {
+                        "name": "internalFrameworks",
+                        "weight": 0.8,
+                        "keywords": ["orbitmesh", "fluxgate"],
+                        "scoring_mode": scoring_mode,
+                    }
+                ],
             },
         )
         result: Final = await router.async_pre_routing_hook(
             model="test-router",
             request_kwargs={},
             messages=[
-                {"role": "system", "content": "orbitmesh"},
-                {"role": "user", "content": "orbitmesh"},
-                {"role": "assistant", "content": "orbitmesh is ready"},
+                {"role": "system", "content": "orbitmesh fluxgate"},
+                {"role": "user", "content": "orbitmesh fluxgate"},
+                {"role": "assistant", "content": "orbitmesh fluxgate is ready"},
                 {"role": "user", "content": current_ask},
-                {"role": "tool", "tool_call_id": "previous", "content": "orbitmesh"},
+                {"role": "tool", "tool_call_id": "previous", "content": "orbitmesh fluxgate"},
             ],
         )
         assert result is not None
         assert result.routing_decision is not None
-        assert ("custom (internalFrameworks)" in result.routing_decision["signals"]) is (current_ask == "orbitmesh")
-        assert result.model == ("top" if current_ask == "orbitmesh" else "cheap")
+        expected_score: Final = (
+            0.0
+            if current_ask == "Hello!"
+            else 0.4
+            if scoring_mode == "match_count" and current_ask == "orbitmesh"
+            else 0.8
+        )
+        assert result.routing_decision["score"] == expected_score
+        assert ("custom (internalFrameworks)" in result.routing_decision["signals"]) is (expected_score > 0)
+        assert result.model == ("cheap" if expected_score == 0 else "strong" if expected_score == 0.4 else "top")
         assert "orbitmesh" not in " ".join(result.routing_decision["signals"])
 
-    def test_custom_patterns_scan_only_the_first_2048_characters(self, mock_router_instance: MagicMock) -> None:
+    @pytest.mark.parametrize("scoring_mode", ("binary", "match_count"))
+    def test_custom_patterns_scan_only_the_first_2048_characters(
+        self, mock_router_instance: MagicMock, scoring_mode: str
+    ) -> None:
         router: Final = ComplexityRouter(
             "test-router",
             mock_router_instance,
-            {"custom_dimensions": [{"name": "late", "weight": 0.7, "patterns": [r"zzz{1,3}"]}]},
+            {
+                "custom_dimensions": [
+                    {
+                        "name": "late",
+                        "weight": 0.7,
+                        "patterns": [r"zzz{1,3}", r"yyy{1,3}"],
+                        "scoring_mode": scoring_mode,
+                    }
+                ]
+            },
         )
+        baseline: Final = ComplexityRouter("test-router", mock_router_instance)
         assert "custom (late)" in router.classify("a" * 2040 + " zzz")[2]
         assert "custom (late)" not in router.classify("a" * 2048 + " zzz")[2]
+        second_hit_past_the_bound: Final = "yyy " + "a" * 2044 + " zzz"
+        contribution: Final = (
+            router.classify(second_hit_past_the_bound)[1] - baseline.classify(second_hit_past_the_bound)[1]
+        )
+        assert contribution == pytest.approx(0.7 if scoring_mode == "binary" else 0.35)
+
+    @pytest.mark.parametrize(
+        "prompt,expected_score",
+        [
+            pytest.param("Hello!", 0.0, id="no-hit"),
+            pytest.param("orbitmesh orbitmesh ORBITMESH again", 0.5, id="one-keyword-repeated"),
+            pytest.param("create table a; CREATE TABLE b; create  table c", 0.5, id="one-pattern-repeated"),
+            pytest.param("orbitmesh and fluxgate", 1.0, id="two-keywords"),
+            pytest.param("orbitmesh then create table t", 1.0, id="keyword-plus-pattern"),
+            pytest.param("create table a; alter table b", 1.0, id="two-patterns"),
+            pytest.param("orbitmesh fluxgate create table a alter table b", 1.0, id="all-matchers"),
+        ],
+    )
+    def test_match_count_grades_distinct_matchers(
+        self, mock_router_instance: MagicMock, prompt: str, expected_score: float
+    ) -> None:
+        dimension: Final = {
+            "name": "graded",
+            "weight": 0.6,
+            "keywords": ["orbitmesh", "ORBITMESH", "fluxgate"],
+            "patterns": [r"\bcreate\s{1,4}table\b", r"\bcreate\s{1,4}table\b", r"\balter\s{1,4}table\b"],
+        }
+        baseline: Final = ComplexityRouter("test-router", mock_router_instance)
+        binary: Final = ComplexityRouter("test-router", mock_router_instance, {"custom_dimensions": [dimension]})
+        graded: Final = ComplexityRouter(
+            "test-router",
+            mock_router_instance,
+            {"custom_dimensions": [{**dimension, "scoring_mode": "match_count"}]},
+        )
+        _, baseline_score, baseline_signals = baseline.classify(prompt)
+        _, binary_score, binary_signals = binary.classify(prompt)
+        _, graded_score, graded_signals = graded.classify(prompt)
+        assert graded_score == pytest.approx(baseline_score + 0.6 * expected_score)
+        assert binary_score == pytest.approx(baseline_score + (0.6 if expected_score else 0.0))
+        expected_signals: Final = [*baseline_signals, *(["custom (graded)"] if expected_score else [])]
+        assert graded_signals == expected_signals
+        assert binary_signals == expected_signals
+
+    def test_scoring_mode_round_trips_and_defaults_to_binary(self) -> None:
+        dimension: Final = {"name": "graded", "weight": 0.6, "keywords": ["orbitmesh"]}
+        legacy: Final = ComplexityRouterConfig.model_validate({"custom_dimensions": [dimension]})
+        graded: Final = ComplexityRouterConfig.model_validate(
+            {"custom_dimensions": [{**dimension, "scoring_mode": "match_count"}]}
+        )
+        assert legacy.custom_dimensions[0].scoring_mode == "binary"
+        assert graded.model_dump(mode="json")["custom_dimensions"][0]["scoring_mode"] == "match_count"
+        assert ComplexityRouterConfig.model_validate(graded.model_dump(mode="json")) == graded
 
     def test_custom_dimensions_router_wide_regex_work_is_capped(self) -> None:
         heavy: Final = {"weight": 0.5, "patterns": ["a?" * 8 + "z"]}
@@ -1511,6 +1595,7 @@ class TestRouterComplexityDeploymentMethods:
     def test_the_shipped_rubric_and_default_prompt_stay_free(self) -> None:
         """Only an operator-written prompt is gated: picking a shipped rubric preset, or writing no
         prompt at all, leaves a router unmetered, so several of them register under a ceiling of one."""
+
         def rubric(model_name: str, model_id: str, preset: str | None) -> dict[str, object]:
             llm_config: dict[str, object] = {"model": "gpt-4o-mini"}
             if preset is not None:
@@ -1647,6 +1732,7 @@ class TestRouterComplexityDeploymentMethods:
     def test_renaming_built_in_tiers_is_not_a_custom_tier_set(self) -> None:
         """tier_labels renames the built-in ladder without defining one, so it stays ungated: two such
         routers register under a ceiling of one."""
+
         def labeled(model_name: str, model_id: str) -> dict[str, object]:
             row = self._router_row(model_name, model_id, "heuristic")
             row["litellm_params"]["complexity_router_config"]["tier_labels"] = {"SIMPLE": "Cheap", "MEDIUM": "Standard"}
@@ -2520,9 +2606,7 @@ class TestLLMClassifier:
         assert outcome.classifier_cost == pytest.approx(1.35e-05)
 
     @pytest.mark.asyncio
-    async def test_aclassify_timeout_does_not_inherit_router_retries_or_fallbacks(
-        self, llm_classifier_config
-    ):
+    async def test_aclassify_timeout_does_not_inherit_router_retries_or_fallbacks(self, llm_classifier_config):
         real_router = Router(
             model_list=[
                 {
@@ -2565,9 +2649,7 @@ class TestLLMClassifier:
         assert real_router.total_calls["openai/mock-backup-classifier"] == 0
 
     @pytest.mark.asyncio
-    async def test_aclassify_enforces_total_classifier_deadline(
-        self, mock_router_instance, llm_classifier_config
-    ):
+    async def test_aclassify_enforces_total_classifier_deadline(self, mock_router_instance, llm_classifier_config):
         cancelled = asyncio.Event()
 
         async def slow_classifier(**_kwargs: object) -> None:
@@ -12414,9 +12496,7 @@ class TestTierHealthFailover:
                     llm_provider="",
                 )
             filtered = (*cooling, *blocked, *excluded)
-            healthy = [
-                {"model_name": model, "model_info": {"id": i}} for i in ids_by_model[model] if i not in filtered
-            ]
+            healthy = [{"model_name": model, "model_info": {"id": i}} for i in ids_by_model[model] if i not in filtered]
             if not healthy:
                 raise RouterRateLimitError(
                     model=model, cooldown_time=60.0, enable_pre_call_checks=False, cooldown_list=[]
@@ -12845,9 +12925,7 @@ class TestTierHealthFailover:
         assert all(probed is not request_kwargs for probed in router.litellm_router_instance.probed_kwargs)
 
     @pytest.mark.asyncio
-    async def test_a_peer_whose_every_deployment_is_over_its_rpm_is_not_a_failover_target(
-        self, mock_router_instance
-    ):
+    async def test_a_peer_whose_every_deployment_is_over_its_rpm_is_not_a_failover_target(self, mock_router_instance):
         """RPM exhaustion is its own verdict from the owner (RouterRateLimitErrorBasic). A peer
         in that state would be rejected downstream, so it cannot be the substitute."""
         from litellm.types.router import RouterRateLimitErrorBasic
@@ -12880,9 +12958,7 @@ class TestTierHealthFailover:
         assert {r.model for r in results} == {"live-c"}
 
     @pytest.mark.asyncio
-    async def test_the_probe_forwards_input_so_window_checks_run_on_input_only_surfaces(
-        self, mock_router_instance
-    ):
+    async def test_the_probe_forwards_input_so_window_checks_run_on_input_only_surfaces(self, mock_router_instance):
         """The Responses API carries its prompt as `input`, never as messages. The owner only
         runs its context-window pre-call check when one of them is present, so dropping `input`
         would silently skip window filtering on that whole surface."""
@@ -12908,9 +12984,7 @@ class TestTierHealthFailover:
         ), "the eligibility probe must forward `input` to the owner"
 
     @pytest.mark.asyncio
-    async def test_a_group_the_router_has_no_deployment_for_is_not_a_failover_target(
-        self, mock_router_instance
-    ):
+    async def test_a_group_the_router_has_no_deployment_for_is_not_a_failover_target(self, mock_router_instance):
         """The owner answers an unconfigured group with BadRequestError. Reading that as live
         would both skip failover off it and let it be chosen as a substitute."""
         router = self._router(
@@ -13099,9 +13173,7 @@ class TestClassifierVision:
         routed as default_fallback on text the request never contained.
         """
         router = self._router(mock_router_instance, vision={"enabled": True})
-        response = await router.async_pre_routing_hook(
-            model="m", request_kwargs={}, messages=self._turn(IMG_PART)
-        )
+        response = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self._turn(IMG_PART))
         assert response.routing_decision["cause"] == "llm_classifier"
         assert response.model == "t-complex"
         assert [block["type"] for block in self._classifier_user_content(mock_router_instance)] == [
@@ -13112,9 +13184,7 @@ class TestClassifierVision:
     @pytest.mark.asyncio
     async def test_image_only_turn_still_falls_back_when_vision_is_off(self, mock_router_instance):
         router = self._router(mock_router_instance, vision={"enabled": False})
-        response = await router.async_pre_routing_hook(
-            model="m", request_kwargs={}, messages=self._turn(IMG_PART)
-        )
+        response = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self._turn(IMG_PART))
         assert response.routing_decision["cause"] == "default_fallback"
         mock_router_instance.acompletion.assert_not_awaited()
 
@@ -13184,9 +13254,7 @@ class TestClassifierVision:
         makes the image the only variable; a margin loose enough to leave the score undecided
         would pass whether or not the guard exists.
         """
-        router = self._router(
-            mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra
-        )
+        router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
         response = await router.async_pre_routing_hook(
             model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
         )
@@ -13200,9 +13268,7 @@ class TestClassifierVision:
         self, mock_router_instance, classifier_type, extra, short_circuit_cause
     ):
         """The negative class: same router, same text, no image, and the scorer still decides."""
-        router = self._router(
-            mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra
-        )
+        router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
         response = await router.async_pre_routing_hook(
             model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}]
         )
diff --git a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py
index f686a62db76..75c115cd3ab 100644
--- a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py
+++ b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py
@@ -7,6 +7,7 @@ from typing import Final
 
 import pytest
 
+from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
 from litellm.router_utils.auto_router_tuning_baseline import (
     DEFAULT_TUNING_FINGERPRINT,
     HEURISTIC_V1_TUNING_FIELDS,
@@ -21,6 +22,33 @@ from litellm.router_utils.auto_router_tuning_baseline import (
 
 _TIERS = {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}
 _ALT_TIERS = {**_TIERS, "COMPLEX": "other-strong"}
+_KEYWORD_DIMENSION: Final = {"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}
+_HISTORICAL_FINGERPRINTS: Final = (
+    ({}, "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"),
+    (
+        {"custom_dimensions": [_KEYWORD_DIMENSION]},
+        "b5c3c3f3be6341a8a16148d68d9067e03f94ed01a0bbfcde7955763042744372",
+    ),
+    (
+        {"custom_dimensions": [{"name": "sqlDdl", "weight": 0.4, "patterns": [r"\bCREATE\s{1,4}TABLE\b"]}]},
+        "814ce0017fc7f60a160b262f658d910e9bdf784e6139a4ba4f1e2657aa203950",
+    ),
+    (
+        {
+            "tiers": _TIERS,
+            "dimension_weights": {"codePresence": 0.3},
+            "custom_dimensions": [
+                {
+                    "name": "internalFrameworks",
+                    "weight": 0.2,
+                    "keywords": ["orbitmesh", "fluxgate"],
+                    "patterns": [r"\bALTER\s{1,4}TABLE\b"],
+                }
+            ],
+        },
+        "38970dc9224e265ab38c89674563d8d0537822591f9239b45251db6f5ca6cc39",
+    ),
+)
 
 
 def _router(
@@ -77,6 +105,27 @@ class TestTuningFingerprint:
     def test_explicit_empty_tier_model_configs_follow_omission(self) -> None:
         assert tuning_fingerprint({"tier_model_configs": {}}) == DEFAULT_TUNING_FINGERPRINT
 
+    @pytest.mark.parametrize(("config", "fingerprint"), _HISTORICAL_FINGERPRINTS)
+    def test_fingerprints_recorded_before_scoring_mode_existed_are_preserved(
+        self, config: Mapping[str, object], fingerprint: str
+    ) -> None:
+        """Literal hashes captured from the merged implementation at 9bc9104102, before CustomDimension.scoring_mode."""
+        assert tuning_fingerprint(config) == fingerprint
+
+    def test_binary_scoring_mode_hashes_like_its_absence(self) -> None:
+        historical: Final = tuning_fingerprint({"custom_dimensions": [_KEYWORD_DIMENSION]})
+        explicit: Final = tuning_fingerprint({"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "binary"}]})
+        reserialized: Final = ComplexityRouterConfig.model_validate(
+            {"custom_dimensions": [_KEYWORD_DIMENSION]}
+        ).model_dump(mode="json", include={"custom_dimensions"})
+        assert reserialized["custom_dimensions"][0]["scoring_mode"] == "binary"
+        assert reserialized["custom_dimensions"][0]["patterns"] == []
+        assert historical == explicit == tuning_fingerprint(reserialized)
+        assert (
+            tuning_fingerprint({"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "match_count"}]})
+            != historical
+        )
+
     def test_tier_model_overrides_change_the_fingerprint(self) -> None:
         plain = tuning_fingerprint({"tiers": {"SIMPLE": "x"}})
         with_override = tuning_fingerprint(
@@ -218,15 +267,18 @@ class TestQuota:
             is None
         )
 
-    def test_custom_dimension_add_edit_and_revert_share_one_quota_slot(self) -> None:
+    @pytest.mark.parametrize(
+        "edit",
+        [
+            pytest.param({"weight": 0.9}, id="weight"),
+            pytest.param({"scoring_mode": "match_count"}, id="scoring-mode"),
+        ],
+    )
+    def test_custom_dimension_add_edit_and_revert_share_one_quota_slot(self, edit: Mapping[str, object]) -> None:
         baselines: Final = snapshot_tuning_baselines(())
         original: Final = _router("a", {})
-        config: Final = {
-            "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}]
-        }
-        edited_config: Final = {
-            "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.9, "keywords": ["orbitmesh"]}]
-        }
+        config: Final = {"custom_dimensions": [_KEYWORD_DIMENSION]}
+        edited_config: Final = {"custom_dimensions": [{**_KEYWORD_DIMENSION, **edit}]}
         added: Final = _router("a", config)
         edited: Final = _router("a", edited_config)
         second: Final = _router("b", config)
@@ -240,6 +292,13 @@ class TestQuota:
         assert mutable_tuned_identities((original,), baselines) == frozenset()
         assert tuning_quota_violation(candidate=second, others=(original,), baselines=baselines, limit=1) is None
 
+    def test_graded_dimension_recorded_at_snapshot_is_its_own_baseline(self) -> None:
+        graded: Final = _router("a", {"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "match_count"}]})
+        baselines: Final = snapshot_tuning_baselines((graded,))
+        assert mutable_tuned_identities((graded,), baselines) == frozenset()
+        reverted_to_binary: Final = _router("a", {"custom_dimensions": [_KEYWORD_DIMENSION]})
+        assert mutable_tuned_identities((reverted_to_binary,), baselines) == {router_identity(graded)}
+
     def test_violation_message_names_the_limit_and_remedy(self) -> None:
         message = tuning_limit_violation(held=2, limit=1)
         assert message is not None
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
index 15cfff01766..111eeebe5a3 100644
--- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
@@ -44,8 +44,9 @@ import {
 } from "./ComplexityRouterConfig";
 
 const DEFAULT_SCORING_EXPLANATION =
-  "The router scores each request across 7 dimensions: token count, code presence, reasoning markers, technical " +
-  "terms, simple indicators, multi-step patterns, and question complexity. The weighted score determines the tier:";
+  "The router scores each request across 7 built-in dimensions: token count, code presence, reasoning markers, technical " +
+  "terms, simple indicators, multi-step patterns, and question complexity, plus any custom dimensions you add. " +
+  "The weighted score determines the tier:";
 
 const HEURISTIC_V2_EXPLANATION =
   "The router estimates success probability for all four tiers with the bundled calibrated model, then selects " +
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
index 2970e14b335..0c12cc0ba1a 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
@@ -1521,14 +1521,16 @@ describe("ComplexityRouterConfig tier editing", () => {
     renderWithProviders(<ComplexityRouterConfig {...baseProps} value={customValue} onEditingTiersChange={vi.fn()} />);
     fireEvent.click(screen.getByText("Advanced: Classification Method"));
     expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument();
-    expect(screen.queryByText("scores each request across 7 dimensions", { exact: false })).not.toBeInTheDocument();
+    expect(
+      screen.queryByText("scores each request across 7 built-in dimensions", { exact: false }),
+    ).not.toBeInTheDocument();
   });
 
   it("keeps the scorer card on a built-in router, whose tiers the score still decides", () => {
     renderWithProviders(<ComplexityRouterConfig {...baseProps} onEditingTiersChange={vi.fn()} />);
     fireEvent.click(screen.getByText("Advanced: Classification Method"));
     expect(screen.getByText("How Classification Works")).toBeInTheDocument();
-    expect(screen.getByText("scores each request across 7 dimensions", { exact: false })).toBeInTheDocument();
+    expect(screen.getByText("scores each request across 7 built-in dimensions", { exact: false })).toBeInTheDocument();
   });
 
   it("says why a custom row is blocked instead of only reddening its border", () => {
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index d7df6ce33bb..de80e714e66 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -50,6 +50,7 @@ import EscalationKeywords from "./EscalationKeywords";
 import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules";
 import SemanticKeywordMatching from "./SemanticKeywordMatching";
 import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs";
+import { type CustomDimensionRow } from "./custom_dimensions";
 import CompressionControls from "./CompressionControls";
 import { type AutoRouterCompressionState, DEFAULT_AUTO_ROUTER_COMPRESSION } from "./buildAutoRouterCompression";
 
@@ -432,6 +433,11 @@ export interface ComplexityRouterConfigValue {
   tier_boundaries?: TierBoundaries;
   token_thresholds?: TokenThresholds;
   dimension_weights?: DimensionWeights;
+  /**
+   * Operator-added scoring dimensions, each carrying its own inline weight. Undefined means the router has
+   * none and keeps the key out of the payload; an empty array is a real "the last row was removed" state.
+   */
+  custom_dimensions?: CustomDimensionRow[];
   /**
    * Score floor the reasoning-marker override must clear. Undefined keeps the key out of the payload, so the
    * floor tracks tier_boundaries.simple_medium; an explicit 0 is a real floor that promotes on the markers alone.
diff --git a/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx
new file mode 100644
index 00000000000..34d8370aafb
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx
@@ -0,0 +1,136 @@
+import { Trash2 } from "lucide-react";
+import { useState } from "react";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Textarea } from "@/components/ui/textarea";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Slider } from "@/components/ui/slider";
+import type { CustomDimensionRow } from "./custom_dimensions";
+
+const SCORING_MODES = [
+  { value: "binary", label: "Binary" },
+  { value: "match_count", label: "Match count" },
+] as const;
+
+interface Props {
+  rows: CustomDimensionRow[];
+  disabled: boolean;
+  onChange: (rows: CustomDimensionRow[]) => void;
+  onWeight: (id: string, weight: number) => void;
+  onAdd: () => void;
+  onRemove: (id: string) => void;
+}
+
+export default function CustomDimensionRows({ rows, disabled, onChange, onWeight, onAdd, onRemove }: Props) {
+  const [draft, setDraft] = useState<{ id: string; raw: string } | null>(null);
+  const update = (id: string, patch: Partial<CustomDimensionRow>) =>
+    onChange(rows.map((row) => (row.id === id ? { ...row, ...patch } : row)));
+  const editWeight = (id: string, raw: string) => {
+    setDraft({ id, raw });
+    if (raw.trim() && Number.isFinite(Number(raw))) onWeight(id, Number(raw));
+  };
+  return (
+    <div className="space-y-4">
+      {rows.map((row, index) => (
+        <fieldset key={row.id} className="min-w-0 space-y-3 rounded-md border p-3">
+          <legend className="float-left text-sm font-semibold">Custom dimension {index + 1}</legend>
+          <div className="flex items-center gap-2">
+            <Button
+              type="button"
+              variant="ghost"
+              size="sm"
+              className="text-destructive hover:text-destructive/80"
+              aria-label={`Remove custom dimension ${index + 1}`}
+              disabled={disabled}
+              onClick={() => onRemove(row.id)}
+            >
+              <Trash2 />
+              Remove
+            </Button>
+          </div>
+          <div className="space-y-1">
+            <Label htmlFor={`${row.id}-name`}>Name</Label>
+            <Input
+              id={`${row.id}-name`}
+              value={row.name}
+              maxLength={64}
+              onChange={(event) => update(row.id, { name: event.target.value })}
+            />
+          </div>
+          <div className="flex flex-wrap items-center gap-3">
+            <Label htmlFor={`${row.id}-weight`}>Weight</Label>
+            <Slider
+              min={0}
+              max={1}
+              step={0.01}
+              disabled={disabled}
+              value={[row.weight]}
+              className="min-w-24 flex-1"
+              aria-label={`${row.name || `Custom dimension ${index + 1}`} weight`}
+              onValueChange={(value) => onWeight(row.id, Array.isArray(value) ? value[0] : value)}
+            />
+            <Input
+              id={`${row.id}-weight`}
+              className="w-24"
+              inputMode="decimal"
+              disabled={disabled}
+              value={draft?.id === row.id ? draft.raw : Number(row.weight.toPrecision(6)).toString()}
+              onBlur={() => setDraft(null)}
+              onChange={(event) => editWeight(row.id, event.target.value)}
+            />
+          </div>
+          <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
+            {(["keywords", "patterns"] as const).map((field) => (
+              <div key={field} className="min-w-0 space-y-1">
+                <Label htmlFor={`${row.id}-${field}`}>
+                  {field === "keywords" ? "Keywords" : "Regex patterns"} (one per line)
+                </Label>
+                <Textarea
+                  id={`${row.id}-${field}`}
+                  rows={2}
+                  value={row[field]?.join("\n") ?? ""}
+                  onChange={(event) =>
+                    update(row.id, { [field]: event.target.value ? event.target.value.split("\n") : [] })
+                  }
+                />
+              </div>
+            ))}
+          </div>
+          <div className="space-y-1">
+            <Label htmlFor={`${row.id}-scoring`}>Scoring</Label>
+            <Select
+              items={SCORING_MODES}
+              value={row.scoring_mode ?? "binary"}
+              onValueChange={(mode) => {
+                if (mode === "binary" || mode === "match_count") update(row.id, { scoring_mode: mode });
+              }}
+            >
+              <SelectTrigger id={`${row.id}-scoring`}>
+                <SelectValue />
+              </SelectTrigger>
+              <SelectContent>
+                {SCORING_MODES.map((mode) => (
+                  <SelectItem key={mode.value} value={mode.value}>
+                    {mode.label}
+                  </SelectItem>
+                ))}
+              </SelectContent>
+            </Select>
+            <p className="text-xs text-muted-foreground">
+              Binary uses the full weight for any hit. Match count uses half for one distinct matcher and full weight
+              for two or more.
+            </p>
+          </div>
+        </fieldset>
+      ))}
+      <Button type="button" variant="outline" size="sm" disabled={disabled || rows.length >= 16} onClick={onAdd}>
+        Add custom dimension
+      </Button>
+      <p className="text-xs text-muted-foreground">
+        Keywords match the current ask. Regex scans its first 2,048 characters and permits bounded single-character
+        repeats up to 64. The proxy validates patterns on save.
+      </p>
+    </div>
+  );
+}
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx
new file mode 100644
index 00000000000..9c4600e3b97
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx
@@ -0,0 +1,90 @@
+import { useState } from "react";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import userEvent from "@testing-library/user-event";
+import { fireEvent, renderWithProviders, screen, testQueryClient, chooseSelectOption } from "../../../tests/test-utils";
+import { SHIPPED_SCORER_DEFAULTS } from "../../../tests/mocks/complexityScorerDefaults";
+import HeuristicScoringConfig from "./HeuristicScoringConfig";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+import { getComplexityScorerDefaults } from "@/components/networking";
+
+vi.mock("@/components/networking", () => ({ getComplexityScorerDefaults: vi.fn() }));
+
+const base: ComplexityRouterConfigValue = {
+  classifier_type: "heuristic",
+  tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
+};
+function Editor({ initial = base }: { initial?: ComplexityRouterConfigValue }) {
+  const [value, setValue] = useState(initial);
+  return (
+    <>
+      <HeuristicScoringConfig value={value} onChange={setValue} />
+      <output aria-label="Draft config">{JSON.stringify(value)}</output>
+    </>
+  );
+}
+const draft = (): ComplexityRouterConfigValue => JSON.parse(screen.getByLabelText("Draft config").textContent!);
+
+beforeEach(() => {
+  testQueryClient.clear();
+  vi.mocked(getComplexityScorerDefaults).mockResolvedValue(SHIPPED_SCORER_DEFAULTS);
+});
+
+describe("combined heuristic editor", () => {
+  it("adds and edits a graded dimension, rebalances builtins, preserves matcher-only edits, and removes it", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(<Editor />);
+    await user.click(screen.getByText("Advanced scoring"));
+    expect(await screen.findByRole("button", { name: "Restore default weights" })).toBeDisabled();
+    await user.click(screen.getByRole("button", { name: "Add custom dimension" }));
+    expect(screen.getByRole("button", { name: "Restore default weights" })).toBeEnabled();
+    expect(draft().dimension_weights?.codePresence).toBeCloseTo(0.27, 12);
+    expect(draft().custom_dimensions?.[0].scoring_mode).toBe("match_count");
+    expect(screen.getByRole("group", { name: "Custom dimension 1" })).toContainElement(
+      screen.getByLabelText("Name", { exact: true }),
+    );
+    fireEvent.change(screen.getByLabelText("Name", { exact: true }), { target: { value: "domain" } });
+    fireEvent.change(screen.getByLabelText("Keywords (one per line)"), { target: { value: "orbitmesh\nfluxgate" } });
+    fireEvent.change(screen.getByLabelText("Weight", { exact: true }), { target: { value: "0.2" } });
+    expect(screen.getByLabelText("Code presence", { exact: true })).toHaveValue("0.24");
+    expect(screen.getByTestId("dimension-weight-total")).toHaveTextContent("total 1.00");
+    const beforeMatchers = draft().dimension_weights;
+    fireEvent.change(screen.getByLabelText("Regex patterns (one per line)"), { target: { value: "abc" } });
+    await chooseSelectOption(user, screen.getByLabelText("Scoring"), "Binary");
+    expect(draft().dimension_weights).toEqual(beforeMatchers);
+    expect(draft().custom_dimensions?.[0].scoring_mode).toBe("binary");
+    await user.click(screen.getByRole("button", { name: "Remove custom dimension 1" }));
+    expect(draft().custom_dimensions).toBeUndefined();
+    expect(draft().dimension_weights?.codePresence).toBeCloseTo(0.3, 12);
+  });
+
+  it("preserves a legacy vector on load and resets both fields only when requested", async () => {
+    const legacy = {
+      ...base,
+      dimension_weights: { codePresence: 0.4 },
+      custom_dimensions: [{ id: "stored-0", name: "domain", weight: 0.7, keywords: ["a"] }],
+    };
+    renderWithProviders(<Editor initial={legacy} />);
+    await userEvent.click(screen.getByText("Advanced scoring"));
+    expect(await screen.findByTestId("dimension-weight-total")).toHaveTextContent("total 1.10");
+    expect(draft()).toEqual(legacy);
+    expect(screen.getByLabelText("Token count", { exact: true })).toHaveValue("0");
+    await userEvent.click(screen.getByRole("button", { name: "Restore default weights" }));
+    expect(draft().custom_dimensions).toBeUndefined();
+    expect(draft().dimension_weights).toBeUndefined();
+  });
+
+  it("drops hidden custom drafts on a fallback-only weight edit so switching back cannot exceed the budget", async () => {
+    const initial: ComplexityRouterConfigValue = {
+      ...base,
+      classifier_type: "llm",
+      classifier_fallback: "heuristic",
+      custom_dimensions: [{ id: "a", name: "domain", weight: 0.7, keywords: ["a"] }],
+    };
+    renderWithProviders(<Editor initial={initial} />);
+    await userEvent.click(screen.getByText("Advanced scoring"));
+    fireEvent.change(await screen.findByLabelText("Code presence", { exact: true }), { target: { value: "0.5" } });
+    expect(draft().custom_dimensions).toBeUndefined();
+    expect(Object.values(draft().dimension_weights!).reduce((sum, weight) => sum + weight, 0)).toBeCloseTo(1, 12);
+    expect(screen.queryByRole("button", { name: "Add custom dimension" })).not.toBeInTheDocument();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx
index f5fb32bb654..61779318fff 100644
--- a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx
@@ -69,13 +69,12 @@ describe("HeuristicScoringConfig", () => {
     expect(onChange).not.toHaveBeenCalled();
   });
 
-  // min and max are inert attributes on a text input, so without the explicit clamp these would persist
-  // a weight of 999, or an infinite boundary, into the router config.
-  it.each([
-    ["Code presence", "999", 1],
-    ["Code presence", "-2", 0],
-    ["Long above", "100000", 100000],
-  ])("clamps %s = %s to %s", async (label, raw, expected) => {
+  it.each(["999", "-2"])("rejects an invalid weight %s instead of silently clamping", async (raw) => {
+    expect(await commit("Code presence", raw)).toBeUndefined();
+    expect(screen.getByRole("alert")).toHaveTextContent("Use a weight from 0 to 1");
+  });
+
+  it.each([["Long above", "100000", 100000]])("clamps %s = %s to %s", async (label, raw, expected) => {
     const next = await commit(label, raw);
     expect(
       { ...next?.dimension_weights, ...next?.token_thresholds }[label === "Long above" ? "complex" : "codePresence"],
@@ -264,6 +263,7 @@ describe("HeuristicScoringConfig degraded states", () => {
     await userEvent.click(screen.getByText("Advanced scoring"));
 
     expect(screen.getByLabelText("Code presence")).toHaveValue("0.5");
+    expect(screen.getByLabelText("Code presence")).toBeDisabled();
     expect(screen.queryByTestId("dimension-weight-total")).not.toBeInTheDocument();
   });
 });
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx
index 540beff8177..4d59ac5bd33 100644
--- a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx
@@ -8,7 +8,14 @@ import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
 import { Slider } from "@/components/ui/slider";
 import { type ComplexityRouterConfigValue, heuristicScoringRole } from "./ComplexityRouterConfig";
-import { dimensionLabel, weightTotal } from "./heuristic_scoring_knobs";
+import {
+  dimensionLabel,
+  effectiveDimensionWeights,
+  rebalanceDimensionWeights,
+  type WeightEdit,
+} from "./heuristic_scoring_knobs";
+import CustomDimensionRows from "./CustomDimensionRows";
+import { customDimensionsError } from "./custom_dimensions";
 
 export type KnobGroup = "tier_boundaries" | "token_thresholds" | "dimension_weights";
 
@@ -54,7 +61,8 @@ const GROUPS: GroupSpec[] = [
   {
     group: "dimension_weights",
     title: "Dimension weights",
-    blurb: "How much each signal contributes to the score. Absolute multipliers, so the total need not be 1.00.",
+    blurb:
+      "Changing a weight rebalances the other built-in and custom weights to total 1.00. Save stores those values. Untouched routers keep their existing weights.",
     min: 0,
     max: 1,
     step: 0.01,
@@ -89,6 +97,19 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
   // The panel owns its own visibility: the scorer does not run at all when an LLM classifier
   // falls back to the default model, so there is nothing here to configure.
   const scorerRuns = heuristicScoringRole(value) !== "never";
+  const customEnabled = heuristicScoringRole(value) === "decides";
+  const customRows = customEnabled ? value.custom_dimensions : undefined;
+  const [weightError, setWeightError] = useState<string | null>(null);
+  const rowError = customDimensionsError(customRows);
+  const changeWeights = (edit: WeightEdit) => {
+    const result = rebalanceDimensionWeights(defaults?.dimension_weights, value.dimension_weights, customRows, edit);
+    if (!result.ok) {
+      setWeightError(result.error);
+      return;
+    }
+    setWeightError(null);
+    onChange({ ...value, dimension_weights: result.dimension_weights, custom_dimensions: result.custom_dimensions });
+  };
 
   // What an untouched override floor follows: the boundary in effect, override included, not the shipped one.
   const trackedFloor: number | undefined = { ...defaults?.tier_boundaries, ...value.tier_boundaries }.simple_medium;
@@ -102,6 +123,10 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
   const commit = (spec: GroupSpec, effective: Record<string, number>, key: string, raw: string) => {
     const parsed = Number(raw);
     if (raw.trim() === "" || !Number.isFinite(parsed)) return;
+    if (spec.group === "dimension_weights") {
+      changeWeights({ type: "set", target: { kind: "builtin", id: key }, weight: parsed });
+      return;
+    }
     const clamped = Math.min(spec.max ?? Infinity, Math.max(spec.min, parsed));
     onChange({
       ...value,
@@ -154,9 +179,19 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                 </div>
               )}
               {GROUPS.map((spec) => {
-                const shipped = defaults?.[spec.group] ?? {};
-                const effective: Record<string, number> = { ...shipped, ...value[spec.group] };
+                const shipped = defaults?.[spec.group] ?? value[spec.group] ?? {};
+                const effective: Record<string, number> =
+                  spec.group === "dimension_weights"
+                    ? effectiveDimensionWeights(shipped, value.dimension_weights)
+                    : { ...shipped, ...value[spec.group] };
+                const total =
+                  Object.values(effective).reduce((sum, weight) => sum + weight, 0) +
+                  (customRows ?? []).reduce((sum, row) => sum + row.weight, 0);
                 const problem = warn(spec.group, effective);
+                const overridden =
+                  value[spec.group] !== undefined || (spec.withSlider && value.custom_dimensions !== undefined);
+                const resettable = spec.withSlider || overridden;
+                const scoringError = spec.withSlider ? weightError || rowError : null;
                 return (
                   <section key={spec.group} className="space-y-2">
                     <div className="flex items-center justify-between">
@@ -166,18 +201,26 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                             alone would state a total that is not the router's. */}
                         {spec.withSlider && defaults !== undefined && (
                           <span className="text-xs text-muted-foreground" data-testid="dimension-weight-total">
-                            total {weightTotal(effective).toFixed(2)}
+                            total {total.toFixed(2)}
                           </span>
                         )}
                       </div>
-                      {value[spec.group] !== undefined && (
+                      {resettable && (
                         <Button
                           type="button"
                           variant="link"
                           size="xs"
-                          onClick={() => onChange({ ...value, [spec.group]: undefined })}
+                          disabled={!overridden}
+                          onClick={() => {
+                            setWeightError(null);
+                            onChange({
+                              ...value,
+                              [spec.group]: undefined,
+                              ...(spec.withSlider && { custom_dimensions: undefined }),
+                            });
+                          }}
                         >
-                          Reset to defaults
+                          {spec.withSlider ? "Restore default weights" : "Reset to defaults"}
                         </Button>
                       )}
                     </div>
@@ -196,6 +239,7 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                               min={spec.min}
                               max={spec.max}
                               step={spec.step}
+                              disabled={defaults === undefined}
                               value={[effective[key]]}
                               onValueChange={(next) =>
                                 commit(spec, effective, key, String(Array.isArray(next) ? next[0] : next))
@@ -209,7 +253,8 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                             type="text"
                             inputMode="decimal"
                             className={spec.withSlider ? "w-24" : "w-28"}
-                            value={draft?.id === id ? draft.raw : String(effective[key])}
+                            disabled={spec.withSlider && defaults === undefined}
+                            value={draft?.id === id ? draft.raw : Number(effective[key].toPrecision(6)).toString()}
                             onChange={(event) => {
                               setDraft({ id, raw: event.target.value });
                               commit(spec, effective, key, event.target.value);
@@ -220,6 +265,28 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                       );
                     })}
 
+                    {spec.withSlider && customEnabled && (
+                      <CustomDimensionRows
+                        rows={customRows ?? []}
+                        disabled={defaults === undefined}
+                        onChange={(rows) => onChange({ ...value, custom_dimensions: rows })}
+                        onWeight={(id, weight) =>
+                          changeWeights({ type: "set", target: { kind: "custom", id }, weight })
+                        }
+                        onAdd={() =>
+                          changeWeights({
+                            type: "add",
+                            row: { id: crypto.randomUUID(), name: "", weight: 0.1, scoring_mode: "match_count" },
+                          })
+                        }
+                        onRemove={(id) => changeWeights({ type: "remove", id })}
+                      />
+                    )}
+                    {scoringError && (
+                      <p className="text-xs text-destructive" role="alert">
+                        {scoringError}
+                      </p>
+                    )}
                     {problem && (
                       <p className="text-xs font-medium text-destructive" role="alert">
                         {problem}
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
index 2d0d6bc2fd8..cf65a093dfb 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
@@ -25,12 +25,14 @@ import ComplexityRouterConfig, {
   ComplexityRouterConfigValue,
   effectiveClassifierType,
   usesLlmClassifier,
+  heuristicScoringRole,
   DEFAULT_ADAPTIVE_WEIGHTS,
   DEFAULT_SESSION_AFFINITY,
   DEFAULT_DEPLOYMENT_AFFINITY,
   DEFAULT_TIER_DISTANCE_PENALTY,
 } from "./ComplexityRouterConfig";
 import { KeywordTierRule } from "./KeywordTierRules";
+import { customDimensionsError } from "./custom_dimensions";
 import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords";
 import {
   type AutoRouterCompressionState,
@@ -139,6 +141,7 @@ export const getSubmitBlockedReason = (
     getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
     getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
     getClassifierModelError(config) ??
+    (heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ??
     getClassifierReasoningEffortError(config, modelInfo) ??
     getReferencedModelsError(referencedModelsParams, availability)
   );
@@ -411,6 +414,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
     tierBoundaries: complexityRouterConfig.tier_boundaries,
     tokenThresholds: complexityRouterConfig.token_thresholds,
     dimensionWeights: complexityRouterConfig.dimension_weights,
+    customDimensions: complexityRouterConfig.custom_dimensions,
     reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score,
     enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation,
     contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer,
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
index baf04822e4f..9973aec7616 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
@@ -704,6 +704,49 @@ describe("buildComplexityRouterConfig scorer knobs", () => {
     expect(buildComplexityRouterConfig(tuned).tier_boundaries).toEqual(BOUNDARIES);
   });
 
+  it("serializes custom rows without changing weights, matcher order, or optional-field absence", () => {
+    const weights = { codePresence: 0.12345678901234568, unknownStoredWeight: 9 };
+    const dimension = { name: "internalFrameworks", weight: 0.3765432109876543, keywords: ["ORBITMESH", "fluxgate"] };
+    const graded = { name: "sqlDdl", weight: 0.5, patterns: ["create table"], scoring_mode: "match_count" as const };
+    const payload = buildComplexityRouterConfig({
+      ...baseParams,
+      dimensionWeights: weights,
+      customDimensions: [
+        { id: "row-1", ...dimension },
+        { id: "row-2", ...graded },
+      ],
+    });
+    expect(payload.dimension_weights).toEqual(weights);
+    expect(payload.custom_dimensions).toEqual([dimension, graded]);
+    expect(buildComplexityRouterConfig(baseParams)).not.toHaveProperty("custom_dimensions");
+    expect(buildComplexityRouterConfig({ ...baseParams, customDimensions: [] }).custom_dimensions).toEqual([]);
+  });
+
+  it.each([
+    ["heuristic", undefined, true],
+    ["heuristic_first", "heuristic", true],
+    ["hybrid", "heuristic", true],
+    ["llm", "heuristic", false],
+    ["custom", "heuristic", false],
+    ["llm", "default_model", false],
+    ["custom", "default_model", false],
+    ["heuristic_v2", undefined, false],
+  ] as const)(
+    "%s with fallback %s only emits custom dimensions when its scorer decides",
+    (classifierType, classifierFallback, emits) => {
+      const dimension = { name: "d", weight: 0.4, keywords: ["orbitmesh"] };
+      const params = {
+        ...baseParams,
+        classifierType,
+        classifierFallback,
+        customDimensions: [{ id: "row", ...dimension }],
+      };
+      const payload = buildComplexityRouterConfig(params);
+      if (emits) expect(payload.custom_dimensions).toEqual([dimension]);
+      else expect(payload).not.toHaveProperty("custom_dimensions");
+    },
+  );
+
   it("drops them when the classifier falls back to the default model and nothing is scored", () => {
     expect(buildComplexityRouterConfig(llmWithDefaultFallback)).not.toHaveProperty("tier_boundaries");
   });
@@ -1060,6 +1103,7 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
       tierBoundaries: { simple_medium: 0.1, medium_complex: 0.25, complex_reasoning: 0.5 },
       tokenThresholds: { short: 1, long: 2 },
       dimensionWeights: { length: 1 },
+      customDimensions: [{ id: "row-1", name: "sqlDdl", weight: 0.4, keywords: ["orbitmesh"] }],
       reasoningOverrideMinScore: 0.5,
       heuristicFirstMaxTier: "SIMPLE",
       hybridBoundaryMargin: 0.03,
@@ -1068,7 +1112,8 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
       stallEscalationWindow: 6,
       stallEscalationRepeatThreshold: 3,
     };
-    const emittingType = key === "heuristic_first_max_tier" ? "heuristic_first" : "llm";
+    // custom_dimensions only ever ship when the scorer decides, so "llm" cannot prove it emits.
+    const emittingType = key === "heuristic_first_max_tier" || key === "custom_dimensions" ? "heuristic_first" : "llm";
     const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType;
     expect(buildComplexityRouterConfig({ ...baseParams, ...loaded, classifierType: typeForKey })).toHaveProperty(key);
     expect(build(loaded)).not.toHaveProperty(key);
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
index 7769fb832fe..d04d48198a8 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
@@ -11,6 +11,7 @@ import {
   tierRowByName,
 } from "./tier_rows";
 import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords";
+import { type CustomDimension, type CustomDimensionRow, serializeCustomDimensions } from "./custom_dimensions";
 import {
   TierModelParams,
   TierModelParamsByTier,
@@ -92,6 +93,7 @@ interface ScorerKnobInputs {
   tierBoundaries: TierBoundaries | undefined;
   tokenThresholds: TokenThresholds | undefined;
   dimensionWeights: DimensionWeights | undefined;
+  customDimensions: CustomDimensionRow[] | undefined;
   reasoningOverrideMinScore: number | undefined;
 }
 
@@ -106,16 +108,23 @@ const scorerKnobPayload = ({
   tierBoundaries,
   tokenThresholds,
   dimensionWeights,
+  customDimensions,
   reasoningOverrideMinScore,
-}: ScorerKnobInputs) =>
-  heuristicScoringRoleFor(classifierType, classifierFallback) === "never"
+}: ScorerKnobInputs) => {
+  const role = heuristicScoringRoleFor(classifierType, classifierFallback);
+  return role === "never"
     ? {}
     : {
         ...(tierBoundaries && { tier_boundaries: tierBoundaries }),
         ...(tokenThresholds && { token_thresholds: tokenThresholds }),
         ...(dimensionWeights && { dimension_weights: dimensionWeights }),
+        // Only a scorer that decides accepts these; the backend rejects them on every other
+        // classifier, so a fallback-only router must not carry rows a switch left behind.
+        ...(role === "decides" &&
+          customDimensions !== undefined && { custom_dimensions: serializeCustomDimensions(customDimensions) }),
         ...(reasoningOverrideMinScore !== undefined && { reasoning_override_min_score: reasoningOverrideMinScore }),
       };
+};
 
 export interface BuildComplexityRouterConfigParams {
   tiers: ComplexityTiers;
@@ -155,6 +164,7 @@ export interface BuildComplexityRouterConfigParams {
   tierBoundaries?: TierBoundaries;
   tokenThresholds?: TokenThresholds;
   dimensionWeights?: DimensionWeights;
+  customDimensions?: CustomDimensionRow[];
   reasoningOverrideMinScore?: number;
   tierModelParams?: TierModelParamsByTier;
   enableContextWindowEscalation?: boolean;
@@ -219,6 +229,7 @@ export interface ComplexityRouterConfigPayload {
   tier_boundaries?: TierBoundaries;
   token_thresholds?: TokenThresholds;
   dimension_weights?: DimensionWeights;
+  custom_dimensions?: CustomDimension[];
   reasoning_override_min_score?: number;
   enable_context_window_escalation?: boolean;
   context_window_escalation_buffer?: number;
@@ -496,6 +507,7 @@ export const buildComplexityRouterConfig = ({
   tierBoundaries,
   tokenThresholds,
   dimensionWeights,
+  customDimensions,
   reasoningOverrideMinScore,
   tierModelParams,
   enableContextWindowEscalation,
@@ -517,6 +529,7 @@ export const buildComplexityRouterConfig = ({
     tierBoundaries,
     tokenThresholds,
     dimensionWeights,
+    customDimensions,
     reasoningOverrideMinScore,
   };
   const scorerKnobs = scorerKnobPayload(scorerInputs);
diff --git a/ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts
new file mode 100644
index 00000000000..3c6060fd083
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts
@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import { customDimensionsError, hydrateCustomDimensions, serializeCustomDimensions } from "./custom_dimensions";
+
+describe("custom dimension drafts", () => {
+  it.each([
+    [[{ name: "domain", weight: 0.2, keywords: [" orbitmesh "] }]],
+    [[{ name: "domain", weight: 0.2, patterns: ["abc"], keywords: [], scoring_mode: "binary" }]],
+    [[{ name: "domain", weight: 0.2, keywords: ["a", "b"], scoring_mode: "match_count" }]],
+    [[]],
+  ])("round-trips stored optional fields and matcher text without normalization: %j", (raw) => {
+    const rows = hydrateCustomDimensions(raw);
+    expect(rows).toBeDefined();
+    expect(serializeCustomDimensions(rows!)).toEqual(raw);
+  });
+
+  it("does not insert a default mode or custom list on load", () => {
+    expect(hydrateCustomDimensions(undefined)).toBeUndefined();
+    const rows = hydrateCustomDimensions([{ name: "domain", weight: 0.2, keywords: ["a"] }])!;
+    expect(rows[0].scoring_mode).toBeUndefined();
+    expect(customDimensionsError(rows)).toBeNull();
+  });
+
+  it.each([
+    { name: "", weight: 0.2, keywords: ["a"] },
+    { name: "codePresence", weight: 0.2, keywords: ["a"] },
+    { name: "bad name", weight: 0.2, keywords: ["a"] },
+    { name: "domain", weight: 0, keywords: ["a"] },
+    { name: "domain", weight: 0.2, keywords: [] },
+    { name: "domain", weight: 0.2, keywords: [" "] },
+    { name: "domain", weight: 0.2, keywords: ["a".repeat(257)] },
+  ])("rejects an invalid draft %j", (row) => {
+    expect(customDimensionsError([{ ...row, id: "draft" }])).not.toBeNull();
+  });
+
+  it("rejects duplicate names and aggregate matcher limits", () => {
+    const row = { id: "a", name: "domain", weight: 0.2, keywords: ["a"] };
+    expect(customDimensionsError([row, { ...row, id: "b", name: "DOMAIN" }])).not.toBeNull();
+    expect(customDimensionsError([{ ...row, keywords: Array(33).fill("a") }])).not.toBeNull();
+    expect(customDimensionsError([{ ...row, keywords: Array(32).fill("a".repeat(256)) }])).not.toBeNull();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts
new file mode 100644
index 00000000000..8cf5a16c303
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts
@@ -0,0 +1,54 @@
+import { z } from "zod";
+import { DIMENSION_LABELS } from "./heuristic_scoring_knobs";
+
+const customDimensionShape = {
+  name: z.string(),
+  weight: z.number(),
+  keywords: z.array(z.string()).optional(),
+  patterns: z.array(z.string()).optional(),
+  scoring_mode: z.enum(["binary", "match_count"]).optional(),
+};
+const customDimensionSchema = z.object(customDimensionShape);
+
+export type CustomDimension = z.infer<typeof customDimensionSchema>;
+export type CustomDimensionRow = CustomDimension & { id: string };
+
+export const hydrateCustomDimensions = (raw: unknown): CustomDimensionRow[] | undefined => {
+  if (raw === undefined) return undefined;
+  const parsed = z.array(customDimensionSchema).safeParse(raw);
+  return parsed.success ? parsed.data.map((row, index) => ({ ...row, id: `stored-${index}` })) : undefined;
+};
+
+export const serializeCustomDimensions = (rows: CustomDimensionRow[]): CustomDimension[] =>
+  rows.map(({ id: _id, ...dimension }) => dimension);
+
+export const customDimensionsError = (
+  rows: CustomDimensionRow[] | undefined,
+  builtinNames: string[] = Object.keys(DIMENSION_LABELS),
+): string | null => {
+  if (!rows) return null;
+  if (rows.length > 16) return "A router can have at most 16 custom dimensions";
+  const names = rows.map((row) => row.name.toLowerCase());
+  for (const [index, row] of rows.entries()) {
+    const prefix = `Custom dimension ${index + 1}: `;
+    if (!/^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(row.name))
+      return (
+        prefix + "use a name starting with a letter, followed by letters, numbers or underscores (64 characters max)"
+      );
+    if (builtinNames.some((name) => name.toLowerCase() === row.name.toLowerCase()))
+      return prefix + "choose a name that is not already a built-in weight";
+    if (names.indexOf(row.name.toLowerCase()) !== index) return prefix + "names must be unique";
+    if (!Number.isFinite(row.weight) || row.weight <= 0 || row.weight > 1)
+      return prefix + "weight must be greater than 0 and at most 1";
+    const matchers = [...(row.keywords ?? []), ...(row.patterns ?? [])];
+    if (!matchers.length || matchers.some((matcher) => !matcher.trim()))
+      return prefix + "add at least one nonblank keyword or pattern";
+    if (
+      matchers.length > 32 ||
+      matchers.some((matcher) => [...matcher].length > 256) ||
+      matchers.reduce((total, matcher) => total + [...matcher].length, 0) > 4096
+    )
+      return prefix + "use at most 32 matchers, 256 characters each and 4096 characters combined";
+  }
+  return null;
+};
diff --git a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts
index ee1d819872b..ea785ddb594 100644
--- a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts
@@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest";
 import { heuristicScoringRoleFor } from "./ComplexityRouterConfig";
 import {
   dimensionLabel,
+  effectiveDimensionWeights,
+  rebalanceDimensionWeights,
   hydrateDimensionWeights,
   hydrateReasoningOverrideMinScore,
   hydrateTierBoundaries,
@@ -68,6 +70,123 @@ describe("hydrating the scorer knobs", () => {
   });
 });
 
+describe("rebalancing the complete weight vector", () => {
+  const defaults = {
+    codePresence: 0.3,
+    reasoningMarkers: 0.25,
+    technicalTerms: 0.25,
+    tokenCount: 0.1,
+    simpleIndicators: 0.05,
+    multiStepPatterns: 0.03,
+    questionComplexity: 0.02,
+  };
+  const row = { id: "domain", name: "domainMarkers", weight: 0.2, keywords: ["orbitmesh"] };
+  const success = (result: ReturnType<typeof rebalanceDimensionWeights>) => {
+    if (!result.ok) throw new Error(result.error);
+    const total =
+      Object.keys(defaults).reduce((sum, name) => sum + result.dimension_weights[name], 0) +
+      (result.custom_dimensions ?? []).reduce((sum, dimension) => sum + dimension.weight, 0);
+    expect(total).toBeCloseTo(1, 12);
+    return result;
+  };
+
+  it("adds a dimension, edits either kind, and redistributes its share on removal", () => {
+    const added = success(rebalanceDimensionWeights(defaults, undefined, undefined, { type: "add", row }));
+    expect(added.dimension_weights.codePresence).toBeCloseTo(0.24, 12);
+    expect(added.custom_dimensions?.[0].weight).toBe(0.2);
+    const edited = success(
+      rebalanceDimensionWeights(defaults, added.dimension_weights, added.custom_dimensions, {
+        type: "set",
+        target: { kind: "custom", id: row.id },
+        weight: 0.4,
+      }),
+    );
+    expect(edited.dimension_weights.codePresence).toBeCloseTo(0.18, 12);
+    const builtin = success(
+      rebalanceDimensionWeights(defaults, edited.dimension_weights, edited.custom_dimensions, {
+        type: "set",
+        target: { kind: "builtin", id: "codePresence" },
+        weight: 0.5,
+      }),
+    );
+    expect(builtin.dimension_weights.codePresence).toBe(0.5);
+    expect(builtin.custom_dimensions?.[0].weight).toBeCloseTo((0.4 * 0.5) / 0.82, 12);
+    const removed = success(
+      rebalanceDimensionWeights(defaults, added.dimension_weights, added.custom_dimensions, {
+        type: "remove",
+        id: row.id,
+      }),
+    );
+    expect(removed.custom_dimensions).toBeUndefined();
+    expect(removed.dimension_weights.codePresence).toBeCloseTo(defaults.codePresence, 12);
+  });
+
+  it("uses zero for omitted keys of an explicit map and preserves ignored keys", () => {
+    expect(effectiveDimensionWeights(defaults, { codePresence: 0.2 }).technicalTerms).toBe(0);
+    expect(effectiveDimensionWeights(defaults, undefined)).toEqual(defaults);
+    const result = success(
+      rebalanceDimensionWeights(defaults, { codePresence: 0.2, unknown: 7 }, undefined, { type: "add", row }),
+    );
+    expect(result.dimension_weights.unknown).toBe(7);
+    expect(result.dimension_weights.codePresence).toBeCloseTo(0.8, 12);
+    expect(result.dimension_weights.technicalTerms).toBe(0);
+  });
+
+  it("distributes an all-zero vector without inventing custom matchers", () => {
+    const result = success(rebalanceDimensionWeights(defaults, {}, undefined, { type: "add", row }));
+    expect(result.dimension_weights.codePresence).toBeCloseTo(0.8 / 7, 12);
+    expect(result.custom_dimensions).toEqual([row]);
+  });
+
+  it("keeps small positive custom weights and full precision", () => {
+    const tiny = { ...row, weight: 1e-8 };
+    const result = success(
+      rebalanceDimensionWeights(defaults, defaults, [tiny], {
+        type: "set",
+        target: { kind: "builtin", id: "codePresence" },
+        weight: 0.123456789,
+      }),
+    );
+    expect(result.dimension_weights.codePresence).toBe(0.123456789);
+    expect(result.custom_dimensions?.[0].weight).toBeGreaterThan(0);
+    expect(result.custom_dimensions?.[0].weight).toBeLessThan(0.01);
+  });
+
+  it.each([0, 1, Number.NaN, -0.1, 1.1])(
+    "rejects invalid or impossible custom weight %s without changing the input",
+    (weight) => {
+      const original = structuredClone(row);
+      const result = rebalanceDimensionWeights(defaults, defaults, [row, { ...row, id: "other" }], {
+        type: "set",
+        target: { kind: "custom", id: row.id },
+        weight,
+      });
+      expect(result.ok).toBe(false);
+      expect(row).toEqual(original);
+    },
+  );
+
+  it("allows one dimension to take the whole budget when no custom sibling needs a share", () => {
+    const result = success(
+      rebalanceDimensionWeights(defaults, defaults, [row], {
+        type: "set",
+        target: { kind: "custom", id: row.id },
+        weight: 1,
+      }),
+    );
+    expect(Object.values(result.dimension_weights).every((weight) => weight === 0)).toBe(true);
+    expect(result.custom_dimensions?.[0].weight).toBe(1);
+  });
+
+  it("refuses missing defaults and invalid stored weights, preserving absence on built-in edits", () => {
+    const edit = { type: "set", target: { kind: "builtin", id: "codePresence" }, weight: 0.2 } as const;
+    expect(rebalanceDimensionWeights(undefined, defaults, undefined, edit).ok).toBe(false);
+    expect(rebalanceDimensionWeights(defaults, { codePresence: -1 }, undefined, edit).ok).toBe(false);
+    expect(success(rebalanceDimensionWeights(defaults, undefined, undefined, edit)).custom_dimensions).toBeUndefined();
+    expect(success(rebalanceDimensionWeights(defaults, undefined, [], edit)).custom_dimensions).toEqual([]);
+  });
+});
+
 describe("heuristicScoringRoleFor", () => {
   it.each([
     ["heuristic", undefined, "decides"],
diff --git a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts
index 1f44a479c60..3de4af92b9c 100644
--- a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts
+++ b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts
@@ -1,3 +1,5 @@
+import type { CustomDimensionRow } from "./custom_dimensions";
+
 export type TierBoundaries = Record<string, number>;
 
 export type TokenThresholds = Record<string, number>;
@@ -27,8 +29,8 @@ const asRecord = (raw: unknown): Record<string, unknown> | undefined =>
 /**
  * Absent means the router is tracking the shipped defaults, so it must hydrate to undefined rather than to
  * a copy of them: hydrating defaults would make an untouched save write them out and pin the router to
- * whatever they were the day the modal was opened. A stored dict is kept exactly as stored, since the
- * backend fills in any key it omits at scoring time.
+ * whatever they were the day the modal was opened. A stored weight map replaces the defaults;
+ * missing dimension names score zero.
  */
 const hydrateNumericMap = (raw: unknown): Record<string, number> | undefined => {
   const stored = asRecord(raw);
@@ -53,3 +55,94 @@ export const hydrateReasoningOverrideMinScore = (raw: unknown): number | undefin
 
 export const weightTotal = (weights: DimensionWeights): number =>
   Math.round(Object.values(weights).reduce((total, weight) => total + weight, 0) * 100) / 100;
+
+export const effectiveDimensionWeights = (
+  defaults: DimensionWeights,
+  stored: DimensionWeights | undefined,
+): DimensionWeights =>
+  Object.fromEntries(
+    Object.keys(defaults).map((name) => [name, stored === undefined ? defaults[name] : stored[name] ?? 0]),
+  );
+
+type WeightTarget = { kind: "builtin" | "custom"; id: string };
+const weightValid = ({ kind, weight }: { kind: WeightTarget["kind"]; weight: number }): boolean => {
+  const inRange = Number.isFinite(weight) && weight >= 0 && weight <= 1;
+  return inRange && (kind === "builtin" || weight > 0);
+};
+export type WeightEdit =
+  | { type: "set"; target: WeightTarget; weight: number }
+  | { type: "add"; row: CustomDimensionRow }
+  | { type: "remove"; id: string };
+type WeightResult =
+  | { ok: true; dimension_weights: DimensionWeights; custom_dimensions: CustomDimensionRow[] | undefined }
+  | { ok: false; error: string };
+
+export const rebalanceDimensionWeights = (
+  defaults: DimensionWeights | undefined,
+  stored: DimensionWeights | undefined,
+  custom: CustomDimensionRow[] | undefined,
+  edit: WeightEdit,
+): WeightResult => {
+  if (!defaults || !Object.keys(defaults).length)
+    return { ok: false, error: "Load the shipped defaults before changing weights" };
+  const builtin = effectiveDimensionWeights(defaults, stored);
+  const rows =
+    edit.type === "add"
+      ? [...(custom ?? []), edit.row]
+      : (custom ?? []).filter((row) => edit.type !== "remove" || row.id !== edit.id);
+  const vector = [
+    ...Object.entries(builtin).map(([id, weight]) => ({ kind: "builtin" as const, id, weight })),
+    ...rows.map(({ id, weight }) => ({ kind: "custom" as const, id, weight })),
+  ];
+  if (!vector.every(weightValid))
+    return {
+      ok: false,
+      error: "Existing weights must be finite and nonnegative; custom weights must be greater than 0 and at most 1",
+    };
+  const target = edit.type === "set" ? edit.target : undefined;
+  const pinned = (entry: WeightTarget) =>
+    edit.type === "add"
+      ? entry.kind === "custom" && entry.id === edit.row.id
+      : entry.kind === target?.kind && entry.id === target.id;
+  if (edit.type === "set" && !vector.some(pinned)) return { ok: false, error: "The dimension is no longer available" };
+  const requestedWeight = (): number => {
+    if (edit.type === "set") return edit.weight;
+    return edit.type === "add" ? edit.row.weight : 0;
+  };
+  const weight = requestedWeight();
+  if (!weightValid({ kind: target?.kind ?? "builtin", weight }))
+    return { ok: false, error: "Use a weight from 0 to 1; custom dimensions must stay greater than 0" };
+  const others = vector.filter((entry) => !pinned(entry));
+  const total = others.reduce((sum, entry) => sum + entry.weight, 0);
+  if (!Number.isFinite(total)) return { ok: false, error: "Existing weights are too large to rebalance" };
+  const remainder = 1 - weight;
+  const builtinCount = others.filter((entry) => entry.kind === "builtin").length;
+  const redistributed = (entry: WeightTarget & { weight: number }): number => {
+    if (pinned(entry)) return weight;
+    if (total > 0) return remainder * (entry.weight / total);
+    return entry.kind === "builtin" ? remainder / builtinCount : 0;
+  };
+  const balanced = vector.map((entry) => ({ ...entry, weight: redistributed(entry) }));
+  const residual = 1 - balanced.reduce((sum, entry) => sum + entry.weight, 0);
+  const receiver = balanced
+    .filter((entry) => entry.kind === "builtin" && !pinned(entry) && entry.weight > 0)
+    .sort((a, b) => b.weight - a.weight)[0];
+  const corrected = balanced.map((entry) =>
+    entry === receiver ? { ...entry, weight: entry.weight + residual } : entry,
+  );
+  const offBudget = Math.abs(corrected.reduce((sum, entry) => sum + entry.weight, 0) - 1) > 1e-12;
+  if (!corrected.every(weightValid) || offBudget)
+    return { ok: false, error: "Leave a positive share for every custom dimension, or remove it first" };
+  const weights = Object.fromEntries(
+    corrected.filter((entry) => entry.kind === "builtin").map(({ id, weight }) => [id, weight]),
+  );
+  const customWeights = new Map(
+    corrected.filter((entry) => entry.kind === "custom").map(({ id, weight }) => [id, weight]),
+  );
+  const emptyRows = edit.type === "remove" ? undefined : custom;
+  return {
+    ok: true,
+    dimension_weights: { ...stored, ...weights },
+    custom_dimensions: rows.length ? rows.map((row) => ({ ...row, weight: customWeights.get(row.id)! })) : emptyRows,
+  };
+};
diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts
index 5e2a32addee..d2cec15d75b 100644
--- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts
+++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts
@@ -136,6 +136,7 @@ export const CUSTOM_TIER_RESTRICTIONS = {
       "tier_boundaries",
       "token_thresholds",
       "dimension_weights",
+      "custom_dimensions",
       "reasoning_override_min_score",
       "custom_technical_keywords",
     ],
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
index 0144dff3498..4a57e9d5e13 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
@@ -63,6 +63,54 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
     expect(result.some_future_backend_key).toEqual({ nested: true });
   });
 
+  describe("custom dimensions", () => {
+    const stored = [
+      { name: "internalFrameworks", weight: 0.7, keywords: ["orbitmesh"] },
+      { name: "sqlDdl", weight: 0.3, patterns: ["create table"], scoring_mode: "match_count" },
+    ];
+    const withDimensions = { ...STORED, custom_dimensions: stored };
+
+    it("round-trips stored rows through hydration and save without dropping or reshaping one", () => {
+      const hydrated = hydrateComplexityRouterConfig(withDimensions, null);
+      expect(hydrated.custom_dimensions).toEqual([
+        { id: "stored-0", ...stored[0] },
+        { id: "stored-1", ...stored[1] },
+      ]);
+      const saved = buildUpdatedComplexityRouterConfig(withDimensions, {
+        ...FORM_VALUE,
+        custom_dimensions: hydrated.custom_dimensions,
+      });
+      expect(saved.custom_dimensions).toEqual(stored);
+    });
+
+    it("is a managed key, so removing the last row does not resurrect the stored dimensions", () => {
+      expect(MANAGED_COMPLEXITY_ROUTER_KEYS.has("custom_dimensions")).toBe(true);
+      const saved = buildUpdatedComplexityRouterConfig(withDimensions, { ...FORM_VALUE, custom_dimensions: [] });
+      expect(saved.custom_dimensions).toEqual([]);
+    });
+
+    it("omits the key entirely when the editor never held rows, so an untouched router gains nothing", () => {
+      expect(hydrateComplexityRouterConfig(STORED, null).custom_dimensions).toBeUndefined();
+      expect(buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE)).not.toHaveProperty("custom_dimensions");
+    });
+
+    it("drops rows a custom tier set forbids rather than sending them to a scorer that never runs", () => {
+      const customTierStored = storedCustomConfig({ custom_dimensions: stored });
+      const saved = buildUpdatedComplexityRouterConfig(customTierStored, {
+        tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
+        classifier_type: "llm" as const,
+        custom_tier_set: {
+          tiers: [
+            { id: "a", name: "CASUAL", definition: "small talk", models: ["gpt-4o-mini"] },
+            { id: "b", name: "AUDIT", definition: "security review", models: ["o1"] },
+          ],
+          fallback_tier_id: "a",
+        },
+      });
+      expect(saved).not.toHaveProperty("custom_dimensions");
+    });
+  });
+
   it("persists an edited keyword rule", () => {
     const result = buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE, undefined, {
       ...hydratedState,
@@ -580,6 +628,7 @@ describe("managed keys survive an untouched open-and-save", () => {
     tier_boundaries: { simple_medium: 0.2, medium_complex: 0.4, complex_reasoning: 0.7 },
     token_thresholds: { simple: 20, complex: 500 },
     dimension_weights: { tokenCount: 0.1 },
+    custom_dimensions: [{ name: "domain", weight: 0.9, keywords: ["orbitmesh"] }],
     reasoning_override_min_score: 0.3,
     enable_context_window_escalation: false,
     context_window_escalation_buffer: 0.9,
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
index 3c0013e267e..b23040ab80c 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
@@ -48,6 +48,7 @@ import {
   hydrateAutoRouterCompression,
 } from "../add_model/buildAutoRouterCompression";
 import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords";
+import { customDimensionsError, hydrateCustomDimensions } from "../add_model/custom_dimensions";
 import {
   hydrateDimensionWeights,
   hydrateReasoningOverrideMinScore,
@@ -61,6 +62,7 @@ import ComplexityRouterConfig, {
   ClassifierType,
   ComplexityRouterConfigValue,
   ComplexityTiers,
+  heuristicScoringRole,
   DEFAULT_ADAPTIVE_WEIGHTS,
   DEFAULT_SESSION_AFFINITY,
   DEFAULT_DEPLOYMENT_AFFINITY,
@@ -109,6 +111,7 @@ export interface StoredComplexityRouterConfig {
   tier_boundaries?: unknown;
   token_thresholds?: unknown;
   dimension_weights?: unknown;
+  custom_dimensions?: unknown;
   reasoning_override_min_score?: unknown;
   session_affinity?: unknown;
   session_affinity_ttl_seconds?: unknown;
@@ -194,6 +197,7 @@ export const hydrateComplexityRouterConfig = (
     tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries),
     token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds),
     dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights),
+    custom_dimensions: hydrateCustomDimensions(parsedConfig.custom_dimensions),
     reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
     session_affinity:
       typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY,
@@ -264,6 +268,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
   "tier_boundaries",
   "token_thresholds",
   "dimension_weights",
+  "custom_dimensions",
   "reasoning_override_min_score",
   "enable_context_window_escalation",
   "context_window_escalation_buffer",
@@ -373,6 +378,7 @@ export const buildUpdatedComplexityRouterConfig = (
     tierBoundaries: value.tier_boundaries,
     tokenThresholds: value.token_thresholds,
     dimensionWeights: value.dimension_weights,
+    customDimensions: value.custom_dimensions,
     reasoningOverrideMinScore: value.reasoning_override_min_score,
     tierModelParams: value.tier_model_params,
     enableContextWindowEscalation: value.enable_context_window_escalation,
@@ -481,7 +487,10 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
             : null) ?? getTierLabelsError(complexityRouterConfig.tier_labels)) ??
       getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ??
       getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ??
-      getClassifierModelError(complexityRouterConfig);
+      getClassifierModelError(complexityRouterConfig) ??
+      (heuristicScoringRole(complexityRouterConfig) === "decides"
+        ? customDimensionsError(complexityRouterConfig.custom_dimensions)
+        : null);
 
   useEffect(() => {
     if (isVisible && modelData) {
@@ -601,7 +610,11 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
         toast.fromError(tierSetError);
         return;
       }
-      const classifierError = getClassifierModelError(complexityRouterConfig);
+      const classifierError =
+        getClassifierModelError(complexityRouterConfig) ??
+        (heuristicScoringRole(complexityRouterConfig) === "decides"
+          ? customDimensionsError(complexityRouterConfig.custom_dimensions)
+          : null);
       if (classifierError) {
         setShowValidationErrors(true);
         toast.fromError(classifierError);
diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
index 344964c4434..5d1c7f73c71 100644
--- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
+++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
@@ -87,6 +87,19 @@ describe("autorouter_presets", () => {
     }
   });
 
+  it("keeps every preset free of custom dimensions, so applying one never adds scoring rows", () => {
+    for (const { complexity_router_config: config } of getAllPresets()) {
+      expect(config.custom_dimensions).toBeUndefined();
+    }
+    const config = getPresetByKey("anthropic_family")!.complexity_router_config;
+    expect(buildPresetPrefill(config, groupsOnly([])).complexityRouterConfig.custom_dimensions).toBeUndefined();
+  });
+
+  it("resets both scoring overrides when the form falls back to an empty prefill", () => {
+    expect(buildEmptyPrefill().complexityRouterConfig.custom_dimensions).toBeUndefined();
+    expect(buildEmptyPrefill().complexityRouterConfig.dimension_weights).toBeUndefined();
+  });
+
   it("carries a preset's session affinity idle window into the prefilled form state", () => {
     const config = getPresetByKey("anthropic_family")!.complexity_router_config;
     const prefill = buildPresetPrefill({ ...config, session_affinity_ttl_seconds: 300 }, groupsOnly([]));
@@ -96,6 +109,21 @@ describe("autorouter_presets", () => {
     ).toBeUndefined();
   });
 
+  it("carries a preset's stored weights and custom dimensions into the prefill without rebalancing them", () => {
+    const config = getPresetByKey("anthropic_family")!.complexity_router_config;
+    const weights = { codePresence: 0.4 };
+    const dimension = { name: "domain", weight: 0.9, keywords: ["orbitmesh"] };
+    const prefill = buildPresetPrefill(
+      { ...config, dimension_weights: weights, custom_dimensions: [dimension] },
+      groupsOnly([]),
+    ).complexityRouterConfig;
+    expect(prefill.dimension_weights).toEqual(weights);
+    expect(prefill.custom_dimensions).toEqual([{ ...dimension, id: "stored-0" }]);
+    const plain = buildPresetPrefill(config, groupsOnly([])).complexityRouterConfig;
+    expect(plain.dimension_weights).toBeUndefined();
+    expect(plain.custom_dimensions).toBeUndefined();
+  });
+
   it("keeps the model-family presets on the heuristic classifier", () => {
     for (const key of ["anthropic_family", "gemini_family", "openai_family"]) {
       expect(getPresetByKey(key)!.complexity_router_config.classifier_type).toBe("heuristic");
diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts
index 2aafbfcfcd9..f2df55fc310 100644
--- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts
+++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts
@@ -13,6 +13,13 @@ import {
 } from "@/components/add_model/ComplexityRouterConfig";
 import { KeywordTierRule } from "@/components/add_model/KeywordTierRules";
 import { hydrateKeywordTierRules } from "@/components/add_model/complexity_router_keywords";
+import { hydrateCustomDimensions } from "@/components/add_model/custom_dimensions";
+import {
+  hydrateDimensionWeights,
+  hydrateTierBoundaries,
+  hydrateTokenThresholds,
+  hydrateReasoningOverrideMinScore,
+} from "@/components/add_model/heuristic_scoring_knobs";
 import {
   TierModelParams,
   TierModelParamsByTier,
@@ -293,6 +300,11 @@ export const buildPresetPrefill = (
       tier_distance_penalty: config.tier_distance_penalty,
       adaptive_eligible: config.adaptive_eligible,
       return_raw_model_name: config.return_raw_model_name,
+      dimension_weights: hydrateDimensionWeights(config.dimension_weights),
+      custom_dimensions: hydrateCustomDimensions(config.custom_dimensions),
+      tier_boundaries: hydrateTierBoundaries(config.tier_boundaries),
+      token_thresholds: hydrateTokenThresholds(config.token_thresholds),
+      reasoning_override_min_score: hydrateReasoningOverrideMinScore(config.reasoning_override_min_score),
       enable_context_window_escalation: config.enable_context_window_escalation,
       context_window_escalation_buffer: config.context_window_escalation_buffer,
     },
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 232b366d8ce..d1431d8956f 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -26591,6 +26591,13 @@ export interface components {
              * @default []
              */
             patterns: string[];
+            /**
+             * Scoring Mode
+             * @description 'binary' scores 1 when any matcher hits. 'match_count' scores 0.5 when one distinct matcher hits and 1 when two or more do; repeated occurrences of one matcher never raise it. Keywords are distinct case-insensitively, patterns by source, and a keyword and a pattern are always distinct from each other.
+             * @default binary
+             * @enum {string}
+             */
+            scoring_mode: "binary" | "match_count";
             /** Weight */
             weight: number;
         };
@@ -34909,7 +34916,7 @@ export interface components {
             context_window_escalation_buffer: number;
             /**
              * Custom Dimensions
-             * @description Named binary dimensions added to the heuristic-v1 score. Each contributes its inline weight once when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters. Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, backreferences and lookarounds are rejected. Conservative work limits include alternation paths, repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. Only heuristic, heuristic_first and hybrid accept this field. Uses the existing heuristic tuning quota.
+             * @description Named dimensions added to the heuristic-v1 score. Each contributes its inline weight once when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters; scoring_mode 'match_count' instead grades half weight for one distinct matcher and full for two or more. Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, backreferences and lookarounds are rejected. Conservative work limits include alternation paths, repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. Only heuristic, heuristic_first and hybrid accept this field. Uses the existing heuristic tuning quota.
              * @default []
              */
             custom_dimensions: components["schemas"]["CustomDimension"][];

From 9e18526887c847415b8333742603daf2bb90bb5f Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:31:07 -0700
Subject: [PATCH 26/27] feat(deploy): metrics sidecar and separate metrics port
 in Helm and Terraform (#40163)

* feat(deploy): expose SSE keepalive, pre-call checks and a metrics sidecar in Helm and Terraform

Typed reliability values on both Helm charts and the AWS/GCP Terraform
modules, a dedicated ClusterIP Service for the separate Prometheus port,
a /health route on the metrics server and dead-worker pruning so the
aggregate does not keep stale multiprocess samples.

Resolves LIT-7142

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(deploy): drop reliability config from Helm and Terraform, keep only the metrics sidecar

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): cover startup pruning of dead workers' live gauges and unsignalable pids

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 docker/component_entrypoint.sh                |   6 +
 helm/litellm-helm/templates/deployment.yaml   |  12 ++
 .../templates/service-metrics.yaml            |  17 ++
 .../templates/servicemonitor.yaml             |   2 +-
 .../tests/metrics_server_tests.yaml           | 106 +++++++++++++
 helm/litellm-helm/values.yaml                 |  10 ++
 helm/litellm/templates/_helpers.tpl           |   2 +
 .../litellm/templates/gateway/deployment.yaml |  53 ++++++-
 .../templates/gateway/service-metrics.yaml    |  18 +++
 helm/litellm/tests/metrics_server_tests.yaml  | 148 ++++++++++++++++++
 helm/litellm/values.yaml                      |  16 ++
 litellm/proxy/prometheus_cleanup.py           |  35 +++++
 litellm/proxy/prometheus_metrics_server.py    |   5 +
 litellm/proxy/proxy_server.py                 |   8 +
 terraform/litellm/aws/README.md               |  16 ++
 terraform/litellm/aws/ecs.tf                  |  54 ++++++-
 .../litellm/aws/examples/default/main.tf      |   3 +
 .../examples/default/terraform.tfvars.example |   7 +
 .../litellm/aws/examples/default/variables.tf |  12 ++
 terraform/litellm/aws/network.tf              |  11 ++
 .../aws/tests/metrics_sidecar.tftest.hcl      | 108 +++++++++++++
 terraform/litellm/aws/variables.tf            |  38 +++++
 .../proxy/proxy_server/test_lifecycle.py      |  25 +++
 .../proxy/test_prometheus_cleanup.py          |  66 +++++++-
 .../proxy/test_prometheus_metrics_server.py   |   9 +-
 .../test_litellm/test_component_entrypoint.py |  53 +++++++
 26 files changed, 830 insertions(+), 10 deletions(-)
 create mode 100644 helm/litellm-helm/templates/service-metrics.yaml
 create mode 100644 helm/litellm-helm/tests/metrics_server_tests.yaml
 create mode 100644 helm/litellm/templates/gateway/service-metrics.yaml
 create mode 100644 helm/litellm/tests/metrics_server_tests.yaml
 create mode 100644 terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl

diff --git a/docker/component_entrypoint.sh b/docker/component_entrypoint.sh
index 413957b9929..173afafe1ad 100755
--- a/docker/component_entrypoint.sh
+++ b/docker/component_entrypoint.sh
@@ -1,5 +1,11 @@
 #!/bin/sh
 
+# stale samples from a previous container incarnation would be summed into the aggregate
+if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then
+    mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
+    rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db
+fi
+
 case "$USE_DDTRACE" in
     [Tt][Rr][Uu][Ee])
         export DD_TRACE_OPENAI_ENABLED="False"
diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml
index 52ffd117535..f7c918a6827 100644
--- a/helm/litellm-helm/templates/deployment.yaml
+++ b/helm/litellm-helm/templates/deployment.yaml
@@ -152,6 +152,13 @@ spec:
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
             {{- end }}
+            {{- if .Values.metricsServer.enabled }}
+            {{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }}
+            {{- fail "metricsServer.port must differ from service.port" }}
+            {{- end }}
+            - name: PROMETHEUS_METRICS_PORT
+              value: {{ .Values.metricsServer.port | quote }}
+            {{- end }}
             {{- if .Values.migrationJob.enabled }}
             # Schema updates are owned by the dedicated migrations Job; skip
             # the proxy's startup `prisma db push` so N replicas don't race
@@ -189,6 +196,11 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+            {{- if .Values.metricsServer.enabled }}
+            - name: metrics
+              containerPort: {{ .Values.metricsServer.port }}
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.livenessProbe.path | quote }}
diff --git a/helm/litellm-helm/templates/service-metrics.yaml b/helm/litellm-helm/templates/service-metrics.yaml
new file mode 100644
index 00000000000..1d23fe39606
--- /dev/null
+++ b/helm/litellm-helm/templates/service-metrics.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.metricsServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "litellm.fullname" . }}-metrics
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.metricsServer.port }}
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "litellm.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/helm/litellm-helm/templates/servicemonitor.yaml b/helm/litellm-helm/templates/servicemonitor.yaml
index 743098deb3f..68083d0da61 100644
--- a/helm/litellm-helm/templates/servicemonitor.yaml
+++ b/helm/litellm-helm/templates/servicemonitor.yaml
@@ -26,7 +26,7 @@ spec:
     {{- toYaml .namespaceSelector.matchNames | nindent 4 }}
     {{- end }}
   endpoints:
-  - port: http
+  - port: {{ ternary "metrics" "http" $.Values.metricsServer.enabled }}
     path: /metrics/
     interval: {{ .interval }}
     scrapeTimeout: {{ .scrapeTimeout }}
diff --git a/helm/litellm-helm/tests/metrics_server_tests.yaml b/helm/litellm-helm/tests/metrics_server_tests.yaml
new file mode 100644
index 00000000000..085d69ac640
--- /dev/null
+++ b/helm/litellm-helm/tests/metrics_server_tests.yaml
@@ -0,0 +1,106 @@
+suite: separate metrics server
+templates:
+  - configmap-litellm.yaml
+  - deployment.yaml
+  - service.yaml
+  - service-metrics.yaml
+  - servicemonitor.yaml
+tests:
+  - it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].ports
+          content:
+            name: metrics
+          any: true
+        template: deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_METRICS_PORT
+          any: true
+        template: deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: service.yaml
+      - hasDocuments:
+          count: 0
+        template: service-metrics.yaml
+
+  - it: should scrape the proxy port when the metrics server is disabled
+    template: servicemonitor.yaml
+    set:
+      serviceMonitor.enabled: true
+    asserts:
+      - equal:
+          path: spec.endpoints[0].port
+          value: http
+
+  - it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor
+    set:
+      metricsServer.enabled: true
+      metricsServer.port: 4101
+      serviceMonitor.enabled: true
+      service.type: LoadBalancer
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_METRICS_PORT
+            value: "4101"
+        template: deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].ports
+          content:
+            name: metrics
+            containerPort: 4101
+            protocol: TCP
+        template: deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: service.yaml
+      - equal:
+          path: spec.type
+          value: LoadBalancer
+        template: service.yaml
+      - equal:
+          path: metadata.name
+          value: RELEASE-NAME-litellm-metrics
+        template: service-metrics.yaml
+      - equal:
+          path: spec.type
+          value: ClusterIP
+        template: service-metrics.yaml
+      - equal:
+          path: spec.ports
+          value:
+            - port: 4101
+              targetPort: metrics
+              protocol: TCP
+              name: metrics
+        template: service-metrics.yaml
+      - equal:
+          path: spec.selector
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+        template: service-metrics.yaml
+      - equal:
+          path: spec.endpoints[0].port
+          value: metrics
+        template: servicemonitor.yaml
+      - equal:
+          path: spec.endpoints[0].path
+          value: /metrics/
+        template: servicemonitor.yaml
+
+  - it: should reject a metrics port equal to the proxy port
+    template: deployment.yaml
+    set:
+      metricsServer.enabled: true
+      metricsServer.port: 4000
+    asserts:
+      - failedTemplate:
+          errorMessage: metricsServer.port must differ from service.port
diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml
index 637be2322e3..8dc7b967e11 100644
--- a/helm/litellm-helm/values.yaml
+++ b/helm/litellm-helm/values.yaml
@@ -180,6 +180,16 @@ proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 
+# Serve Prometheus /metrics from a separate process (PROMETHEUS_METRICS_PORT)
+# so a scrape never runs on an inference worker. Adds a `metrics` port to the
+# container and a dedicated ClusterIP `<release>-metrics` Service, and the
+# ServiceMonitor scrapes it instead of the proxy port. The separate port has
+# no virtual-key auth: keep it off public ingress. Needs the proxy image
+# v1.101.0 or newer.
+metricsServer:
+  enabled: false
+  port: 4001
+
 resources:
   {}
   # Unset by default so the chart installs on small clusters such as Minikube, and so an
diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl
index be6b9093f53..c459512c7b9 100644
--- a/helm/litellm/templates/_helpers.tpl
+++ b/helm/litellm/templates/_helpers.tpl
@@ -441,3 +441,5 @@ ImplementationSpecific
 {{- .pathType -}}
 {{- end -}}
 {{- end -}}
+
+{{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml
index 5030ba2c9dc..9cb6b07e77b 100644
--- a/helm/litellm/templates/gateway/deployment.yaml
+++ b/helm/litellm/templates/gateway/deployment.yaml
@@ -64,14 +64,25 @@ spec:
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
             {{- end }}
+            {{- if .Values.gateway.metricsServer.enabled }}
+            {{- if eq (int .Values.gateway.metricsServer.port) 4000 }}
+            {{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }}
+            {{- end }}
+            - name: PROMETHEUS_MULTIPROC_DIR
+              value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
+            {{- if .Values.gateway.metricsServer.enabled }}
+            - name: prometheus-multiproc
+              mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+            {{- end }}
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
             {{- end }}
@@ -97,16 +108,54 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.gateway.resources | nindent 12 }}
+        {{- if .Values.gateway.metricsServer.enabled }}
+        - name: metrics
+          image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
+          {{- with .Values.gateway.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command:
+            - python
+            - -m
+            - litellm.proxy.prometheus_metrics_server
+            - --port
+            - {{ .Values.gateway.metricsServer.port | quote }}
+          env:
+            - name: PROMETHEUS_MULTIPROC_DIR
+              value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.gateway.metricsServer.port }}
+              protocol: TCP
+          volumeMounts:
+            - name: prometheus-multiproc
+              mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+          readinessProbe:
+            tcpSocket: { port: metrics }
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket: { port: metrics }
+            periodSeconds: 15
+            failureThreshold: 6
+          resources:
+            {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }}
+        {{- end }}
       {{- with .Values.gateway.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
+        {{- if .Values.gateway.metricsServer.enabled }}
+        - name: prometheus-multiproc
+          emptyDir: {}
+        {{- end }}
         {{- if .Values.billingMetrics.enabled }}
         {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
         {{- end }}
diff --git a/helm/litellm/templates/gateway/service-metrics.yaml b/helm/litellm/templates/gateway/service-metrics.yaml
new file mode 100644
index 00000000000..ad9bc05a9fd
--- /dev/null
+++ b/helm/litellm/templates/gateway/service-metrics.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.gateway.enabled .Values.gateway.metricsServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "litellm.gateway.fullname" . }}-metrics
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: gateway
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.gateway.metricsServer.port }}
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "litellm.gateway.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/helm/litellm/tests/metrics_server_tests.yaml b/helm/litellm/tests/metrics_server_tests.yaml
new file mode 100644
index 00000000000..0e7d9d9e9ee
--- /dev/null
+++ b/helm/litellm/tests/metrics_server_tests.yaml
@@ -0,0 +1,148 @@
+suite: test gateway metrics sidecar
+templates:
+  - gateway/configmap.yaml
+  - gateway/deployment.yaml
+  - gateway/service.yaml
+  - gateway/service-metrics.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: adds no sidecar, volume, env or service port when the metrics server is off
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 1
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_MULTIPROC_DIR
+          any: true
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: prometheus-multiproc
+          any: true
+        template: gateway/deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: gateway/service.yaml
+      - hasDocuments:
+          count: 0
+        template: gateway/service-metrics.yaml
+
+  - it: runs the metrics server as a sidecar over a shared multiproc dir and exposes it on a ClusterIP metrics service
+    set:
+      gateway.metricsServer.enabled: true
+      gateway.metricsServer.port: 4101
+      gateway.service.type: LoadBalancer
+      gateway.image.tag: v1.101.0
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_MULTIPROC_DIR
+            value: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: prometheus-multiproc
+            mountPath: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: metrics
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].image
+          value: ghcr.io/berriai/litellm-gateway:v1.101.0
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].command
+          value:
+            - python
+            - -m
+            - litellm.proxy.prometheus_metrics_server
+            - --port
+            - "4101"
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].env
+          value:
+            - name: PROMETHEUS_MULTIPROC_DIR
+              value: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].ports
+          value:
+            - name: metrics
+              containerPort: 4101
+              protocol: TCP
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].volumeMounts
+          value:
+            - name: prometheus-multiproc
+              mountPath: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].readinessProbe.tcpSocket.port
+          value: metrics
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].livenessProbe.tcpSocket.port
+          value: metrics
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].resources.requests.cpu
+          value: 50m
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: prometheus-multiproc
+            emptyDir: {}
+        template: gateway/deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: gateway/service.yaml
+      - equal:
+          path: spec.type
+          value: LoadBalancer
+        template: gateway/service.yaml
+      - equal:
+          path: metadata.name
+          value: RELEASE-NAME-litellm-gateway-metrics
+        template: gateway/service-metrics.yaml
+      - equal:
+          path: spec.type
+          value: ClusterIP
+        template: gateway/service-metrics.yaml
+      - equal:
+          path: spec.ports
+          value:
+            - port: 4101
+              targetPort: metrics
+              protocol: TCP
+              name: metrics
+        template: gateway/service-metrics.yaml
+      - equal:
+          path: spec.selector
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: gateway
+        template: gateway/service-metrics.yaml
+
+  - it: rejects a metrics port equal to the gateway port
+    template: gateway/deployment.yaml
+    set:
+      gateway.metricsServer.enabled: true
+      gateway.metricsServer.port: 4000
+    asserts:
+      - failedTemplate:
+          errorMessage: gateway.metricsServer.port must differ from the gateway port 4000
diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml
index 6c9fb9440c7..b5d535c992d 100644
--- a/helm/litellm/values.yaml
+++ b/helm/litellm/values.yaml
@@ -268,6 +268,22 @@ gateway:
   config:
     create: true
     proxy_config: {}
+  # Serve Prometheus /metrics from a `metrics` sidecar container (same image,
+  # `python -m litellm.proxy.prometheus_metrics_server`) that aggregates the
+  # workers' PROMETHEUS_MULTIPROC_DIR samples over a shared emptyDir, so a
+  # scrape never runs on an inference worker. Adds a `metrics` port to the pod
+  # and a dedicated ClusterIP `<gateway>-metrics` Service; point your scrape
+  # config at it. The port has no virtual-key auth: keep it off public ingress.
+  # Needs the gateway image v1.101.0 or newer.
+  metricsServer:
+    enabled: false
+    port: 4001
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+      limits:
+        memory: 512Mi
   image:
     repository: ghcr.io/berriai/litellm-gateway
     tag: ""              # defaults to .Chart.AppVersion
diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py
index d9827723887..c65aaeedfaa 100644
--- a/litellm/proxy/prometheus_cleanup.py
+++ b/litellm/proxy/prometheus_cleanup.py
@@ -8,10 +8,13 @@ from __future__ import annotations
 
 import glob
 import os
+import re
 from typing import Final
 
 from litellm._logging import verbose_proxy_logger
 
+_LIVE_GAUGE_PID: Final = re.compile(r"gauge_live[a-z]*_(\d+)\.db$")
+
 
 def wipe_directory(directory: str) -> None:
     """Delete all .db files in the directory. Called once before workers fork."""
@@ -38,3 +41,35 @@ def mark_worker_exit(worker_pid: int) -> None:
         verbose_proxy_logger.info("Prometheus cleanup: marked worker %s as dead", worker_pid)
     except Exception as e:
         verbose_proxy_logger.warning("Failed to mark prometheus worker %s as dead: %s", worker_pid, e)
+
+
+def _is_running(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def mark_dead_workers(directory: str) -> tuple[int, ...]:
+    """Drop the live-gauge files of workers that no longer exist and return their pids.
+
+    Uvicorn's multi-worker supervisor has no exit hook, so a replacement worker calls this at startup; without it
+    a crashed worker's in-flight gauges stay in the aggregate forever.
+    """
+    owners: Final = frozenset(
+        int(match.group(1))
+        for match in map(_LIVE_GAUGE_PID.search, glob.glob(os.path.join(directory, "gauge_live*_*.db")))
+        if match is not None
+    )
+    dead: Final = tuple(sorted(pid for pid in owners if pid != os.getpid() and not _is_running(pid)))
+    if not dead:
+        return dead
+    from prometheus_client import multiprocess
+
+    for pid in dead:
+        multiprocess.mark_process_dead(pid, path=directory)
+    verbose_proxy_logger.info("Prometheus cleanup: marked dead workers %s in %s", dead, directory)
+    return dead
diff --git a/litellm/proxy/prometheus_metrics_server.py b/litellm/proxy/prometheus_metrics_server.py
index 4a9651d62e1..01479f9dd41 100644
--- a/litellm/proxy/prometheus_metrics_server.py
+++ b/litellm/proxy/prometheus_metrics_server.py
@@ -30,6 +30,7 @@ from litellm.integrations.prometheus_metrics_endpoint import make_metrics_asgi_a
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 METRICS_PATH: Final = "/metrics"
+HEALTH_PATH: Final = "/health"
 PID_HEADER: Final = "x-litellm-metrics-pid"
 _PARENT_POLL_INTERVAL_SECONDS: Final = 1.0
 _STARTUP_TIMEOUT_SECONDS: Final = 30.0
@@ -77,6 +78,10 @@ def build_metrics_app(multiproc_dir: str) -> FastAPI:
     app: Final = FastAPI(title="LiteLLM Prometheus metrics", docs_url=None, redoc_url=None, openapi_url=None)
     app.mount(METRICS_PATH, _add_pid_header(make_metrics_asgi_app(registry)))
 
+    @app.get(HEALTH_PATH)
+    def health() -> dict[str, str]:
+        return {"status": "healthy", "multiproc_dir": multiproc_dir}
+
     return app
 
 
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 1d6baf5fded..38c4c2e96c9 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -631,6 +631,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     router as pass_through_router,
 )
+from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit
 from litellm.proxy.public_endpoints import router as public_endpoints_router
 from litellm.proxy.public_endpoints.public_v1 import router as public_v1_router
 from litellm.proxy.rag_endpoints.endpoints import router as rag_router
@@ -1059,6 +1060,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
 
     init_verbose_loggers()
 
+    prometheus_multiproc_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+    if prometheus_multiproc_dir:
+        mark_dead_workers(prometheus_multiproc_dir)
+
     ## RUN WORKER STARTUP HOOKS (e.g., gflags initialization) ##
     _startup_hooks_env: Final = os.environ.get("LITELLM_WORKER_STARTUP_HOOKS", "")
     if _startup_hooks_env:
@@ -1365,6 +1370,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
 
     await proxy_shutdown_event(worker_heartbeat=worker_heartbeat)
 
+    if prometheus_multiproc_dir:
+        mark_worker_exit(os.getpid())
+
 
 def _generate_stable_operation_id(route: "APIRoute") -> str:
     operation_id = re.sub(r"\W", "_", f"{route.name}{route.path_format}")
diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md
index 389027bf5ca..923a4ca2da8 100644
--- a/terraform/litellm/aws/README.md
+++ b/terraform/litellm/aws/README.md
@@ -242,6 +242,22 @@ this with `litellm_license`. To tune the export cadence, set
 `LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` /
 `backend_extra_env`
 
+### Prometheus metrics sidecar
+
+`gateway_metrics_port` adds a `metrics` sidecar
+(`python -m litellm.proxy.prometheus_metrics_server`) to the gateway task that
+aggregates the workers' samples over a shared task volume, so a scrape never
+runs on an inference worker. The ALB never routes to that port and the tasks
+security group only opens it to `gateway_metrics_scrape_cidrs`. Needs
+`gateway_image` v1.101.0 or newer. See
+[Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) for the
+metrics themselves.
+
+```hcl
+gateway_metrics_port         = 4001
+gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
+```
+
 ## Tenant deployment
 
 Every resource the stack creates is named `${tenant}-litellm-${env}` (or
diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf
index 2fb1ca08205..aa2c3d558e2 100644
--- a/terraform/litellm/aws/ecs.tf
+++ b/terraform/litellm/aws/ecs.tf
@@ -212,6 +212,45 @@ locals {
   # pull the config from S3 first, so the command goes through `sh -c`;
   # otherwise we keep the image's ENTRYPOINT and only override `command`.
   gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}"
+
+  metrics_enabled       = var.gateway_metrics_port != null
+  metrics_multiproc_dir = "/tmp/litellm_prometheus_multiproc"
+  metrics_volume        = "prometheus-multiproc"
+  metrics_env           = local.metrics_enabled ? [{ name = "PROMETHEUS_MULTIPROC_DIR", value = local.metrics_multiproc_dir }] : []
+  metrics_mount_points  = local.metrics_enabled ? [{ sourceVolume = local.metrics_volume, containerPath = local.metrics_multiproc_dir }] : []
+  metrics_health_cmd    = "import socket; socket.create_connection(('127.0.0.1', ${coalesce(var.gateway_metrics_port, 0)}), timeout=2).close()"
+
+  gateway_metrics_container = local.metrics_enabled ? [
+    {
+      name       = "metrics"
+      image      = var.gateway_image
+      essential  = false
+      entryPoint = ["python", "-m", "litellm.proxy.prometheus_metrics_server"]
+      command    = ["--port", tostring(var.gateway_metrics_port)]
+
+      portMappings = [{ containerPort = var.gateway_metrics_port, protocol = "tcp" }]
+      environment  = local.metrics_env
+      mountPoints  = local.metrics_mount_points
+
+      healthCheck = {
+        command     = ["CMD", "python", "-c", local.metrics_health_cmd]
+        interval    = 30
+        timeout     = 5
+        retries     = 3
+        startPeriod = 30
+      }
+
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          awslogs-group         = aws_cloudwatch_log_group.gateway.name
+          awslogs-region        = var.region
+          awslogs-stream-prefix = "metrics"
+        }
+      }
+    }
+  ] : []
+
   backend_uvicorn_args = "--host 0.0.0.0 --port 4001"
 
   gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac"
@@ -269,7 +308,7 @@ resource "aws_ecs_task_definition" "gateway" {
   execution_role_arn       = aws_iam_role.task_execution.arn
   task_role_arn            = aws_iam_role.task.arn
 
-  container_definitions = jsonencode([
+  container_definitions = jsonencode(concat([
     merge(
       {
         name      = "gateway"
@@ -283,8 +322,10 @@ resource "aws_ecs_task_definition" "gateway" {
           local.billing_metrics_env,
           local.gateway_extra_env_list,
           local.proxy_config_env,
+          local.metrics_env,
         )
-        secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list)
+        secrets     = concat(local.shared_secrets, local.gateway_extra_secrets_list)
+        mountPoints = local.metrics_mount_points
 
         # Container-level healthCheck intentionally omitted — the wolfi
         # runtime image doesn't ship curl/wget. The ALB target group polls
@@ -301,7 +342,14 @@ resource "aws_ecs_task_definition" "gateway" {
       },
       local.gateway_proxy_overrides,
     )
-  ])
+  ], local.gateway_metrics_container))
+
+  dynamic "volume" {
+    for_each = local.metrics_enabled ? [1] : []
+    content {
+      name = local.metrics_volume
+    }
+  }
 
   tags = local.tags
 }
diff --git a/terraform/litellm/aws/examples/default/main.tf b/terraform/litellm/aws/examples/default/main.tf
index 2eeaf6adb50..115003f7ddd 100644
--- a/terraform/litellm/aws/examples/default/main.tf
+++ b/terraform/litellm/aws/examples/default/main.tf
@@ -48,4 +48,7 @@ module "litellm" {
   backend_extra_env     = var.backend_extra_env
   gateway_extra_secrets = var.gateway_extra_secrets
   backend_extra_secrets = var.backend_extra_secrets
+
+  gateway_metrics_port         = var.gateway_metrics_port
+  gateway_metrics_scrape_cidrs = var.gateway_metrics_scrape_cidrs
 }
diff --git a/terraform/litellm/aws/examples/default/terraform.tfvars.example b/terraform/litellm/aws/examples/default/terraform.tfvars.example
index 59301ea6aa5..880ecf56555 100644
--- a/terraform/litellm/aws/examples/default/terraform.tfvars.example
+++ b/terraform/litellm/aws/examples/default/terraform.tfvars.example
@@ -102,6 +102,13 @@ env    = "stage"
 #   }
 # }
 
+# ---------- Prometheus metrics sidecar ----------
+# Serve /metrics from a sidecar in the gateway task instead of the inference
+# workers. The port is not behind the ALB and has no auth: open it only to
+# your Prometheus subnets.
+# gateway_metrics_port         = 4001
+# gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
+
 # ---------- Extra env / secrets ----------
 # Plain-text env vars (non-sensitive). Land directly in the ECS task def.
 # gateway_extra_env = {
diff --git a/terraform/litellm/aws/examples/default/variables.tf b/terraform/litellm/aws/examples/default/variables.tf
index d8ab56b13af..f8140266fca 100644
--- a/terraform/litellm/aws/examples/default/variables.tf
+++ b/terraform/litellm/aws/examples/default/variables.tf
@@ -158,3 +158,15 @@ variable "backend_extra_secrets" {
   type        = map(string)
   default     = {}
 }
+
+variable "gateway_metrics_port" {
+  description = "Port for the Prometheus metrics sidecar in the gateway task. Null keeps /metrics on the gateway port only."
+  type        = number
+  default     = null
+}
+
+variable "gateway_metrics_scrape_cidrs" {
+  description = "CIDRs allowed to scrape gateway_metrics_port."
+  type        = list(string)
+  default     = []
+}
diff --git a/terraform/litellm/aws/network.tf b/terraform/litellm/aws/network.tf
index 4563eefbba5..c54949b4a59 100644
--- a/terraform/litellm/aws/network.tf
+++ b/terraform/litellm/aws/network.tf
@@ -156,6 +156,17 @@ resource "aws_security_group" "tasks" {
     security_groups = [aws_security_group.alb.id]
   }
 
+  dynamic "ingress" {
+    for_each = local.metrics_enabled && length(var.gateway_metrics_scrape_cidrs) > 0 ? [1] : []
+    content {
+      description = "Prometheus scrapers to the gateway metrics sidecar"
+      from_port   = var.gateway_metrics_port
+      to_port     = var.gateway_metrics_port
+      protocol    = "tcp"
+      cidr_blocks = var.gateway_metrics_scrape_cidrs
+    }
+  }
+
   egress {
     description = "All egress (LLM providers, RDS, Redis)"
     from_port   = 0
diff --git a/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl
new file mode 100644
index 00000000000..de839d7e5be
--- /dev/null
+++ b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl
@@ -0,0 +1,108 @@
+# Plan-only coverage for the Prometheus metrics sidecar wiring. Offline via
+# mock_provider, same as byo_infrastructure.tftest.hcl.
+
+mock_provider "aws" {
+  mock_data "aws_iam_policy_document" {
+    defaults = {
+      json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
+    }
+  }
+}
+mock_provider "random" {}
+
+variables {
+  region              = "us-east-1"
+  tenant              = "acme"
+  env                 = "test"
+  allow_plaintext_alb = true
+  azs                 = ["us-east-1a", "us-east-1b"]
+}
+
+run "defaults_change_nothing" {
+  command = plan
+
+  assert {
+    condition = alltrue([
+      length(local.gateway_metrics_container) == 0,
+      length(local.metrics_env) == 0,
+      length(local.metrics_mount_points) == 0,
+      length([for r in aws_security_group.tasks.ingress : r if r.description == "Prometheus scrapers to the gateway metrics sidecar"]) == 0,
+    ])
+    error_message = "The metrics sidecar, its env, its volume, and its security-group rule must all be absent by default."
+  }
+}
+
+run "metrics_port_adds_a_sidecar_volume_and_scrape_rule" {
+  command = plan
+
+  variables {
+    gateway_metrics_port         = 9464
+    gateway_metrics_scrape_cidrs = ["10.20.0.0/16"]
+  }
+
+  assert {
+    condition     = length(local.metrics_env) == 1 && local.metrics_env[0].name == "PROMETHEUS_MULTIPROC_DIR" && local.metrics_env[0].value == "/tmp/litellm_prometheus_multiproc"
+    error_message = "The gateway workers must write multiprocess samples to the shared dir."
+  }
+
+  assert {
+    condition     = length(local.metrics_mount_points) == 1 && local.metrics_mount_points[0].sourceVolume == "prometheus-multiproc" && local.metrics_mount_points[0].containerPath == "/tmp/litellm_prometheus_multiproc"
+    error_message = "Gateway and sidecar must mount the same task volume at the multiproc dir."
+  }
+
+  assert {
+    condition = alltrue([
+      length(local.gateway_metrics_container) == 1,
+      local.gateway_metrics_container[0].name == "metrics",
+      local.gateway_metrics_container[0].essential == false,
+      join(" ", local.gateway_metrics_container[0].entryPoint) == "python -m litellm.proxy.prometheus_metrics_server",
+      join(" ", local.gateway_metrics_container[0].command) == "--port 9464",
+      one(local.gateway_metrics_container[0].portMappings).containerPort == 9464,
+      one(local.gateway_metrics_container[0].environment).value == "/tmp/litellm_prometheus_multiproc",
+      one(local.gateway_metrics_container[0].mountPoints).sourceVolume == "prometheus-multiproc",
+      strcontains(local.gateway_metrics_container[0].healthCheck.command[3], "9464"),
+    ])
+    error_message = "The metrics sidecar must run prometheus_metrics_server on the configured port, share the multiproc volume, and health-check that port."
+  }
+
+  assert {
+    condition     = length(aws_ecs_task_definition.gateway.volume) == 1 && one(aws_ecs_task_definition.gateway.volume).name == "prometheus-multiproc"
+    error_message = "The gateway task must declare the multiproc volume."
+  }
+
+  assert {
+    condition = length([
+      for r in aws_security_group.tasks.ingress : r
+      if r.from_port == 9464 && r.to_port == 9464 && r.protocol == "tcp" && r.cidr_blocks == tolist(["10.20.0.0/16"])
+    ]) == 1
+    error_message = "The scrape CIDRs must be allowed to reach the metrics port on the tasks security group."
+  }
+
+  assert {
+    condition     = aws_lb_target_group.gateway.port == 4000 && one(aws_ecs_service.gateway.load_balancer).container_port == 4000
+    error_message = "The ALB must keep targeting the gateway port only; the metrics port is never load balanced."
+  }
+}
+
+run "metrics_port_without_scrape_cidrs_opens_nothing" {
+  command = plan
+
+  variables {
+    gateway_metrics_port = 9464
+  }
+
+  assert {
+    condition     = length(local.gateway_metrics_container) == 1 && length([for r in aws_security_group.tasks.ingress : r if r.from_port == 9464]) == 0
+    error_message = "Without scrape CIDRs the sidecar runs but the metrics port stays closed to everything but the ALB group."
+  }
+}
+
+run "metrics_port_may_not_reuse_the_gateway_port" {
+  command = plan
+
+  variables {
+    gateway_metrics_port = 4000
+  }
+
+  expect_failures = [var.gateway_metrics_port]
+}
diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf
index 522138953d6..667f6db63c9 100644
--- a/terraform/litellm/aws/variables.tf
+++ b/terraform/litellm/aws/variables.tf
@@ -549,6 +549,44 @@ variable "proxy_config" {
   default     = {}
 }
 
+# ---------- Prometheus metrics sidecar ----------
+
+variable "gateway_metrics_port" {
+  description = <<-EOT
+    Serve Prometheus /metrics from a `metrics` sidecar container in the
+    gateway task on this port (1-65535, not 4000), so a scrape never runs on
+    an inference worker. The sidecar runs the gateway image with
+    `python -m litellm.proxy.prometheus_metrics_server` and aggregates the
+    workers' PROMETHEUS_MULTIPROC_DIR samples over a task volume. Null (the
+    default) leaves /metrics on the gateway port only. The sidecar port has
+    no virtual-key auth and is not routed through the ALB; open it to your
+    scrapers with gateway_metrics_scrape_cidrs. Needs gateway_image v1.101.0
+    or newer.
+  EOT
+  type        = number
+  default     = null
+
+  validation {
+    condition     = var.gateway_metrics_port == null || (var.gateway_metrics_port >= 1 && var.gateway_metrics_port <= 65535 && var.gateway_metrics_port != 4000)
+    error_message = "gateway_metrics_port must be between 1 and 65535 and must not be 4000 (the gateway port)."
+  }
+}
+
+variable "gateway_metrics_scrape_cidrs" {
+  description = <<-EOT
+    CIDR blocks allowed to reach gateway_metrics_port on the gateway tasks
+    (your Prometheus or collector subnets). Empty by default, so only the
+    ALB can reach the tasks. Ignored when gateway_metrics_port is null.
+  EOT
+  type        = list(string)
+  default     = []
+
+  validation {
+    condition     = alltrue([for c in var.gateway_metrics_scrape_cidrs : can(cidrnetmask(c))])
+    error_message = "gateway_metrics_scrape_cidrs must contain valid IPv4 CIDR blocks."
+  }
+}
+
 variable "log_retention_days" {
   description = "CloudWatch log retention for the three services."
   type        = number
diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py
index 7db85dc6943..dafe686c4f2 100644
--- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py
+++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py
@@ -22,6 +22,7 @@ import inspect
 import json
 import logging
 import os
+import subprocess
 from collections.abc import Awaitable, Callable
 from typing import List, Optional, Union
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -749,6 +750,30 @@ async def test_proxy_startup_event_invalid_missing_app_arg_raises():
             pass
 
 
+@pytest.mark.asyncio
+async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path):
+    """With PROMETHEUS_MULTIPROC_DIR set, a booting worker drops the live-gauge files of pids that no longer
+    exist, so a crashed worker's in-flight samples leave the aggregate as soon as its replacement starts."""
+    exited = subprocess.Popen(["true"])
+    assert exited.wait(timeout=30) == 0
+    stale = tmp_path / f"gauge_livesum_{exited.pid}.db"
+    stale.touch()
+    counter = tmp_path / f"counter_{exited.pid}.db"
+    counter.touch()
+
+    clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}
+    clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path)
+    with patch.dict(os.environ, clean_env, clear=True):
+        try:
+            async with proxy_startup_event(app=None):
+                pass
+        except Exception:
+            pass
+
+    assert not stale.exists()
+    assert counter.exists()
+
+
 def test_otel_global_provider_published_after_callback_init():
     """The OTel V2 global-provider publish must run after callback
     initialization in ``proxy_startup_event``.
diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py
index 93b9b694c2c..6a1b95c51ff 100644
--- a/tests/test_litellm/proxy/test_prometheus_cleanup.py
+++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py
@@ -6,13 +6,77 @@ ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir.
 from __future__ import annotations
 
 import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Final
 from unittest.mock import patch
 
 import pytest
+from prometheus_client import CollectorRegistry, multiprocess
 
-from litellm.proxy.prometheus_cleanup import mark_worker_exit, wipe_directory
+from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit, wipe_directory
 from litellm.proxy.proxy_cli import ProxyInitializationHelpers
 
+_WORKER: Final = """
+import sys, time
+from prometheus_client import Gauge
+Gauge("litellm_in_flight", "", multiprocess_mode="livesum").set(float(sys.argv[1]))
+print("ready", flush=True)
+if sys.argv[2] == "stay":
+    time.sleep(120)
+"""
+
+
+def _spawn_worker(directory: Path, in_flight: str, lifetime: str) -> subprocess.Popen[str]:
+    env = {**os.environ, "PROMETHEUS_MULTIPROC_DIR": str(directory)}
+    worker = subprocess.Popen(
+        [sys.executable, "-c", _WORKER, in_flight, lifetime], env=env, stdout=subprocess.PIPE, text=True
+    )
+    assert worker.stdout is not None and worker.stdout.readline() == "ready\n"
+    return worker
+
+
+def _livesum(directory: Path) -> float:
+    registry = CollectorRegistry()
+    multiprocess.MultiProcessCollector(registry, path=str(directory))
+    value = registry.get_sample_value("litellm_in_flight")
+    return 0.0 if value is None else value
+
+
+class TestMarkDeadWorkers:
+    def test_drops_live_gauges_of_exited_workers_and_keeps_running_ones(self, tmp_path: Path) -> None:
+        """A worker that died mid-request leaves its livesum file behind; the replacement worker's startup prune
+        must remove exactly that file so the aggregate stops counting requests nobody is serving."""
+        dead = _spawn_worker(tmp_path, "3", "exit")
+        assert dead.wait(timeout=30) == 0
+        alive = _spawn_worker(tmp_path, "2", "stay")
+        try:
+            assert (tmp_path / f"gauge_livesum_{dead.pid}.db").exists()
+            assert _livesum(tmp_path) == 5.0
+
+            assert mark_dead_workers(str(tmp_path)) == (dead.pid,)
+
+            assert not (tmp_path / f"gauge_livesum_{dead.pid}.db").exists()
+            assert (tmp_path / f"gauge_livesum_{alive.pid}.db").exists()
+            assert _livesum(tmp_path) == 2.0
+            assert mark_dead_workers(str(tmp_path)) == ()
+        finally:
+            alive.kill()
+            alive.wait(timeout=30)
+
+    def test_leaves_counters_of_exited_workers_alone(self, tmp_path: Path) -> None:
+        (tmp_path / "counter_424242.db").touch()
+        (tmp_path / "histogram_424242.db").touch()
+        assert mark_dead_workers(str(tmp_path)) == ()
+        assert sorted(p.name for p in tmp_path.glob("*.db")) == ["counter_424242.db", "histogram_424242.db"]
+
+    def test_keeps_live_gauges_of_workers_it_may_not_signal(self, tmp_path: Path) -> None:
+        """Signal 0 to pid 1 raises PermissionError for an unprivileged proxy; that pid is alive, not dead."""
+        (tmp_path / "gauge_livesum_1.db").touch()
+        assert mark_dead_workers(str(tmp_path)) == ()
+        assert (tmp_path / "gauge_livesum_1.db").exists()
+
 
 class TestWipeDirectory:
     def test_deletes_all_db_files(self, tmp_path):
diff --git a/tests/test_litellm/proxy/test_prometheus_metrics_server.py b/tests/test_litellm/proxy/test_prometheus_metrics_server.py
index fc1fa381fa4..e2f461e61a6 100644
--- a/tests/test_litellm/proxy/test_prometheus_metrics_server.py
+++ b/tests/test_litellm/proxy/test_prometheus_metrics_server.py
@@ -1,5 +1,5 @@
-"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose only /metrics, and follow its
-parent's lifetime.
+"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose /metrics plus a probe-friendly
+/health, and follow its parent's lifetime.
 
 Everything here runs on loopback against a child of this test process; no LLM keys or external network.
 """
@@ -91,7 +91,10 @@ def test_metrics_app_aggregates_multiproc_dir_and_reports_pid(tmp_path: Path, mo
     assert metrics.headers[PID_HEADER] == str(os.getpid())
     assert 'litellm_requests_metric_total{model="gpt-5"} 5.0' in metrics.text
 
-    assert client.get("/health").status_code == 404
+    health: Final = client.get("/health")
+    assert health.status_code == 200
+    assert health.json() == {"status": "healthy", "multiproc_dir": str(tmp_path)}
+    assert client.get("/docs").status_code == 404
 
     empty: Final = TestClient(build_metrics_app(str(other_dir))).get("/metrics")
     assert empty.status_code == 200
diff --git a/tests/test_litellm/test_component_entrypoint.py b/tests/test_litellm/test_component_entrypoint.py
index b0969e2c694..09837d2b233 100644
--- a/tests/test_litellm/test_component_entrypoint.py
+++ b/tests/test_litellm/test_component_entrypoint.py
@@ -223,6 +223,59 @@ def test_gating_matches_the_monolithic_entrypoint_and_get_secret_bool(
     assert monolith[1] == ("args=litellm --port 4000" if traced else "args=--port 4000")
 
 
+def test_wipes_the_prometheus_multiproc_dir_before_uvicorn_forks(tmp_path: Path) -> None:
+    """A restarted container inherits the emptyDir of its predecessor, whose worker pids it may reuse, so the
+    stale .db files must be gone before any worker opens the one carrying its own pid."""
+    multiproc_dir = tmp_path / "multiproc"
+    multiproc_dir.mkdir()
+    (multiproc_dir / "gauge_livesum_7.db").write_bytes(b"stale")
+    (multiproc_dir / "counter_7.db").write_bytes(b"stale")
+    (multiproc_dir / "keep.txt").write_text("not a sample")
+
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    _write_stubs(bin_dir, ("uvicorn",))
+    record = tmp_path / "record.txt"
+    env = {
+        **os.environ,
+        "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+        "RECORD": str(record),
+        "PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir),
+    }
+    env.pop("USE_DDTRACE", None)
+    result = subprocess.run(
+        ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"],
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}"
+    assert sorted(p.name for p in multiproc_dir.iterdir()) == ["keep.txt"]
+    assert record.read_text().splitlines()[0] == "exec=uvicorn"
+
+
+def test_creates_a_missing_prometheus_multiproc_dir(tmp_path: Path) -> None:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    _write_stubs(bin_dir, ("uvicorn",))
+    missing = tmp_path / "multiproc"
+    env = {
+        **os.environ,
+        "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+        "RECORD": str(tmp_path / "record.txt"),
+        "PROMETHEUS_MULTIPROC_DIR": str(missing),
+    }
+    env.pop("USE_DDTRACE", None)
+    result = subprocess.run(
+        ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"], env=env, capture_output=True, text=True
+    )
+
+    assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}"
+    assert missing.is_dir()
+
+
 def _copied_script(dockerfile: Path, image_path: str) -> Path:
     """Resolve the repo file a Dockerfile `COPY`s to `image_path`, so tests run what the image ships."""
     matches = _COPY_RE.findall(dockerfile.read_text())

From 0175c7da1c9c3d6f71c77c52436f250d43c3a8a3 Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Tue, 8 Sep 2026 13:31:28 -0700
Subject: [PATCH 27/27] feat(router): resolve max_tokens to the tier model's
 ceiling on auto-routed requests (#40209)

A client behind an auto-router sends one max_tokens for every tier, so a value
sized for the smallest tier starves a bigger tier's thinking budget and a value
sized for the biggest is rejected by the smallest. After the complexity router
picks a tier, its per-tier litellm_params now carry max_tokens set to the
smallest max_output_tokens across that tier model's deployments (model_info,
then the cost map), applied the same way a per-tier reasoning_effort already
is, on every routing exit including plan mode, the empty-ask default and the
classifier fallback. The router seam collapses whichever ceiling alias a tier
carries onto the surface's own name, so one tier max_tokens reaches chat,
/v1/messages and /v1/responses alike, drops the caller's other carriers of the
same setting before the merge, and stamps the caller's original once so a
fallback into a group no tier owns gets it back instead of a ceiling sized for
the tier that failed.

Proxy-level reservations were sized from the caller's cap before routing, so a
raised cap left them short. Both owners now re-validate at the deployment hook:
the v3 limiter tops up its combined-TPM and project-OTPM reservations to the
final cap or writes the admitted cap back, and the budget limiter re-estimates
on the chosen deployment and grows the reservation or writes the admitted cap
back. An auto-router alias also reserves budget at its priciest tier model now
instead of pricing to zero.

An explicit per-tier max_tokens, max_completion_tokens or max_output_tokens
still wins, and max_tokens_from_tier_model: false forwards the caller's value
unchanged.
---
 litellm/constants.py                          |   2 +
 litellm/proxy/common_utils/callback_utils.py  |   2 +
 litellm/proxy/litellm_pre_call_utils.py       |   5 +-
 litellm/router.py                             | 150 ++++++-
 .../complexity_router/complexity_router.py    |  57 ++-
 .../complexity_router/config.py               |  14 +
 .../proxy/test_litellm_pre_call_utils.py      |   8 +-
 .../router_strategy/test_complexity_router.py | 392 +++++++++++++++++-
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   6 +
 9 files changed, 602 insertions(+), 34 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 8ef9523a60a..8a6a044ff7b 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1461,6 +1461,8 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata"
 OLD_LITELLM_METADATA_FIELD: Final = "metadata"
 RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
 SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
+OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens", "max_output_tokens"})
+CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py
index 39e74d2c8bd..d9d56778405 100644
--- a/litellm/proxy/common_utils/callback_utils.py
+++ b/litellm/proxy/common_utils/callback_utils.py
@@ -10,6 +10,7 @@ import litellm
 from litellm import get_secret
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     PRE_CALL_EXECUTED_GUARDRAILS_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
@@ -507,6 +508,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset(
         PRE_CALL_EXECUTED_GUARDRAILS_KEY,
         SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
         CONSUMED_REQUEST_TAGS_METADATA_KEY,
+        CLIENT_OUTPUT_CEILING_METADATA_KEY,
         "disable_global_guardrails",
         "disable_global_guardrail",
         "opted_out_global_guardrails",
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 3c186e19829..893174dab03 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -18,6 +18,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
 from litellm._uuid import uuid
 from litellm.constants import (
+    CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
     LITELLM_PROXY_MASTER_KEY_ALIAS,
@@ -325,7 +326,9 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg
 # ``attempted_fallbacks`` and ``original_model_group`` are written by the router
 # and read by spend logs as fact; a client value has no legitimate meaning and no
 # key or team setting keeps it, so the strip is never gated.
-_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset({"attempted_fallbacks", "original_model_group"})
+_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
+    {"attempted_fallbacks", "original_model_group", CLIENT_OUTPUT_CEILING_METADATA_KEY}
+)
 _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"
 
 # Request fields whose value, when URL-valued, becomes the outbound destination
diff --git a/litellm/router.py b/litellm/router.py
index 843b916d90f..cf62ce1eb2d 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -21,7 +21,16 @@ import time
 import traceback
 import weakref
 from collections import defaultdict
-from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Iterator, Mapping, Sequence
+from collections.abc import (
+    AsyncGenerator,
+    AsyncIterator,
+    Callable,
+    Generator,
+    Iterator,
+    Mapping,
+    MutableMapping,
+    Sequence,
+)
 from functools import lru_cache, partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast
@@ -45,12 +54,14 @@ from litellm.caching.caching import (
     RedisClusterCache,
 )
 from litellm.constants import (
+    CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
     DEFAULT_HEALTH_CHECK_INTERVAL,
     DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
     DEFAULT_MAX_LRU_CACHE_SIZE,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    OUTPUT_TOKEN_CEILING_PARAMS,
     RUNTIME_UPDATABLE_ROUTER_SETTINGS,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
 )
@@ -648,6 +659,18 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper):
         self.fallback_headers_adopted = True
 
 
+def as_output_cap(value: object) -> int | None:
+    """A client-sent output cap coerced to an int: ints, floats and numeric strings, never bools
+    or negatives."""
+    if isinstance(value, bool) or not isinstance(value, (int, float, str)):
+        return None
+    try:
+        cap: Final = int(float(value))
+    except (ValueError, OverflowError):
+        return None
+    return cap if cap >= 0 else None
+
+
 class Router:
     model_names: set = set()
     cache_responses: bool | None = False
@@ -12606,7 +12629,83 @@ class Router:
             request_kwargs.pop(carrier, None)
 
     @staticmethod
-    def _drop_client_effort_carriers_a_tier_pin_supersedes(
+    def _tier_ceiling_under_the_surface_name(
+        tier_litellm_params: Mapping[str, object], responses_call: bool
+    ) -> Mapping[str, object]:
+        """``max_tokens``, ``max_completion_tokens`` and ``max_output_tokens`` are one
+        ceiling under three names, and each surface reads exactly one of them: the
+        Responses bridge builds its internal ``max_tokens`` from ``max_output_tokens``
+        and would overwrite the tier's, chat and /v1/messages never read
+        ``max_output_tokens``, and litellm already renames ``max_tokens`` to
+        ``max_completion_tokens`` for the OpenAI models that require it. Collapse
+        whatever the tier carries onto the surface's own name, preferring a value the
+        operator already wrote under that name."""
+        surface_key: Final = "max_output_tokens" if responses_call else "max_tokens"
+        carried: Final = tuple(
+            key
+            for key in (surface_key, "max_tokens", "max_completion_tokens", "max_output_tokens")
+            if key in tier_litellm_params
+        )
+        if not carried:
+            return tier_litellm_params
+        return MappingProxyType(
+            {
+                **{k: v for k, v in tier_litellm_params.items() if k not in OUTPUT_TOKEN_CEILING_PARAMS},
+                surface_key: tier_litellm_params[carried[0]],
+            }
+        )
+
+    def _pin_tier_params_onto_request(
+        self,
+        model: str,
+        tier_litellm_params: Mapping[str, object] | None,
+        request_kwargs: dict,
+        responses_call: bool,
+    ) -> bool:
+        """Apply a routing strategy's per-tier litellm_params on top of the request and report
+        whether they pinned an output ceiling, so the caller can hand the request its own ceiling
+        back on a routing pass that pins none."""
+        if not tier_litellm_params:
+            return False
+        accepted_tier_params: Final = self._tier_params_the_target_accepts(model, tier_litellm_params, request_kwargs)
+        surface_tier_params: Final = self._tier_ceiling_under_the_surface_name(
+            accepted_tier_params, responses_call=responses_call
+        )
+        self._drop_client_carriers_a_tier_pin_supersedes(request_kwargs, surface_tier_params)
+        request_kwargs.update(surface_tier_params)
+        return not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(surface_tier_params)
+
+    @staticmethod
+    def _restore_client_ceiling_no_tier_pins(request_kwargs: MutableMapping[str, object]) -> None:
+        """A model-group fallback re-enters routing with the kwargs an earlier auto-router pass
+        already rewrote, so a ceiling sized for that pass's tier would ride onto a group no tier
+        chose. When this pass pins none, hand the request back exactly the carriers the caller
+        sent, which the first pinning pass stamped. The stamp lives in a metadata bucket a
+        caller can also write, so the proxy strips the key at ingestion and this read takes
+        nothing but the three ceiling carriers as integers: no other key ever reaches kwargs."""
+        stamped: Final = next(
+            (
+                bucket.get(CLIENT_OUTPUT_CEILING_METADATA_KEY)
+                for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata"))
+                if isinstance(bucket, dict) and CLIENT_OUTPUT_CEILING_METADATA_KEY in bucket
+            ),
+            None,
+        )
+        if not isinstance(stamped, dict):
+            return
+        callers_ceiling: Final = MappingProxyType(
+            {
+                carrier: cap
+                for carrier, value in stamped.items()
+                if carrier in OUTPUT_TOKEN_CEILING_PARAMS and (cap := as_output_cap(value)) is not None
+            }
+        )
+        for carrier in OUTPUT_TOKEN_CEILING_PARAMS:
+            request_kwargs.pop(carrier, None)
+        request_kwargs.update(callers_ceiling)
+
+    @staticmethod
+    def _drop_client_carriers_a_tier_pin_supersedes(
         request_kwargs: dict[str, object],
         tier_litellm_params: Mapping[str, object],
     ) -> None:
@@ -12616,7 +12715,22 @@ class Router:
         the ``reasoning_effort`` alias, so a pinned effort only reaches the wire
         if the client's other encodings are removed before the merge. Non-effort
         fields a carrier also holds (``output_config.format``,
-        ``reasoning.summary``) are kept."""
+        ``reasoning.summary``) are kept. An output ceiling has the same shape:
+        ``max_tokens``, ``max_completion_tokens`` and ``max_output_tokens`` are
+        one setting under three names, and a provider handed two of them either
+        rejects the request or picks one by iteration order."""
+        if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_litellm_params):
+            _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs)
+            metadata_bucket.setdefault(
+                CLIENT_OUTPUT_CEILING_METADATA_KEY,
+                {
+                    carrier: request_kwargs[carrier]
+                    for carrier in OUTPUT_TOKEN_CEILING_PARAMS
+                    if carrier in request_kwargs
+                },
+            )
+            for carrier in OUTPUT_TOKEN_CEILING_PARAMS:
+                request_kwargs.pop(carrier, None)
         if "reasoning_effort" not in tier_litellm_params:
             return
         request_kwargs.pop("thinking", None)
@@ -12657,6 +12771,7 @@ class Router:
             # Execute Pre-Routing Hooks
             # this hook can modify the model, messages before the routing decision is made
             #########################################################
+            responses_call: Final = input is not None and messages is None
             pre_routing_hook_response: Final = await self.async_pre_routing_hook(
                 model=model,
                 request_kwargs=request_kwargs,
@@ -12668,12 +12783,14 @@ class Router:
                 model = pre_routing_hook_response.model
                 messages = pre_routing_hook_response.messages
                 record_pre_routing_selection(request_kwargs, model)
-                if pre_routing_hook_response.litellm_params:
-                    accepted_tier_params: Final = self._tier_params_the_target_accepts(
-                        model, pre_routing_hook_response.litellm_params, request_kwargs
-                    )
-                    self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
-                    request_kwargs.update(accepted_tier_params)
+            tier_pins_ceiling: Final = self._pin_tier_params_onto_request(
+                model=model,
+                tier_litellm_params=pre_routing_hook_response.litellm_params if pre_routing_hook_response else None,
+                request_kwargs=request_kwargs,
+                responses_call=responses_call,
+            )
+            if not tier_pins_ceiling:
+                self._restore_client_ceiling_no_tier_pins(request_kwargs)
             #########################################################
 
             # Resolve the strategy and logger AFTER the pre-routing hook, since
@@ -12773,6 +12890,7 @@ class Router:
             parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs)
 
             # 1. Execute pre-routing hook
+            responses_call: Final = input is not None and messages is None
             pre_routing_hook_response: Final = await self.async_pre_routing_hook(
                 model=model,
                 request_kwargs=request_kwargs,
@@ -12784,12 +12902,14 @@ class Router:
                 model = pre_routing_hook_response.model
                 messages = pre_routing_hook_response.messages
                 record_pre_routing_selection(request_kwargs, model)
-                if pre_routing_hook_response.litellm_params:
-                    accepted_tier_params: Final = self._tier_params_the_target_accepts(
-                        model, pre_routing_hook_response.litellm_params, request_kwargs
-                    )
-                    self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
-                    request_kwargs.update(accepted_tier_params)
+            tier_pins_ceiling: Final = self._pin_tier_params_onto_request(
+                model=model,
+                tier_litellm_params=pre_routing_hook_response.litellm_params if pre_routing_hook_response else None,
+                request_kwargs=request_kwargs,
+                responses_call=responses_call,
+            )
+            if not tier_pins_ceiling:
+                self._restore_client_ceiling_no_tier_pins(request_kwargs)
 
             # 2. Get healthy deployments
             healthy_deployments: Final = await self.async_get_healthy_deployments(
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index ae1f82fc40b..f9b4f50c15c 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -31,6 +31,7 @@ from litellm._logging import verbose_router_logger
 from litellm.constants import (
     EMPTY_MAPPING,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    OUTPUT_TOKEN_CEILING_PARAMS,
     RETURN_RAW_MODEL_NAME_METADATA_KEY,
     SESSION_ID_GENERATED_METADATA_KEY,
 )
@@ -2110,11 +2111,15 @@ class ComplexityRouter(CustomLogger):
         raise ValueError(f"No model configured for tier {tier_key} and no default_model set")
 
     def _litellm_params_for_model(self, tier: ComplexityTier | str | None, model: str) -> Mapping[str, object]:
-        if tier is None:
-            return MappingProxyType({})
-        entries: Final = self.config.tier_model_configs.get(_tier_name(tier), ())
+        entries: Final = self.config.tier_model_configs.get(_tier_name(tier), ()) if tier is not None else ()
         entry: Final = next((candidate for candidate in entries if candidate.model_name == model), None)
-        return entry.litellm_params if entry is not None else MappingProxyType({})
+        explicit: Final = entry.litellm_params if entry is not None else MappingProxyType({})
+        if not self.config.max_tokens_from_tier_model or not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(explicit):
+            return explicit
+        ceiling: Final = self._group_output_ceiling(model)
+        if ceiling is None:
+            return explicit
+        return MappingProxyType({**explicit, "max_tokens": ceiling})
 
     @staticmethod
     def _pick_from_tier_value(model: str | Sequence[str], tier_key: str) -> str:
@@ -2449,12 +2454,15 @@ class ComplexityRouter(CustomLogger):
         return name if self.config.has_custom_tiers else ComplexityTier(name)
 
     def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None:
+        return self._deployment_limit(group, deployment, "max_input_tokens")
+
+    def _deployment_limit(
+        self, group: str, deployment: Mapping[str, object], key: Literal["max_input_tokens", "max_output_tokens"]
+    ) -> int | None:
         from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
 
         deployment_model_info: Final = deployment.get("model_info")
-        declared: Final = (
-            deployment_model_info.get("max_input_tokens") if isinstance(deployment_model_info, Mapping) else None
-        )
+        declared: Final = deployment_model_info.get(key) if isinstance(deployment_model_info, Mapping) else None
         if isinstance(declared, int):
             return declared
         litellm_params: Final = deployment.get("litellm_params")
@@ -2471,18 +2479,34 @@ class ComplexityRouter(CustomLogger):
                 deployment=cast(dict, deployment),  # cast-ok: router deployments are plain dicts
                 received_model_name=group,
             )
-            window: Final = model_info.get("max_input_tokens")
+            limit: Final = model_info.get(key)
         except Exception:  # noqa: BLE001  # best-effort: an unmappable deployment must not hide the others
             return None
-        return window if isinstance(window, int) else None
+        return limit if isinstance(limit, int) else None
+
+    def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]:
+        list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
+        deployments: Final = list_models(model_name=group) if callable(list_models) else None
+        return tuple(deployments) if isinstance(deployments, list) else ()
+
+    def _group_output_ceiling(self, group: str) -> int | None:
+        """Smallest max_output_tokens across the group's deployments, or None when any deployment
+        declares none: the core router picks within the group without a fit check, and a ceiling
+        above an unmapped member's real limit is a provider 400 on that member."""
+        deployments: Final = self._group_deployments(group)
+        ceilings: Final = tuple(
+            ceiling
+            for deployment in deployments
+            if (ceiling := self._deployment_limit(group, deployment, "max_output_tokens")) is not None
+        )
+        return min(ceilings) if ceilings and len(ceilings) == len(deployments) else None
 
     def _group_window_facts(self, group: str) -> tuple[int | None, bool]:
         """(smallest declared context window across the group's deployments, whether any deployment
         declares none). The core router picks a deployment within the group without a fit check, so
         the group is only as safe as its smallest member."""
-        list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
-        deployments: Final = list_models(model_name=group) if callable(list_models) else None
-        if not isinstance(deployments, list) or not deployments:
+        deployments: Final = self._group_deployments(group)
+        if not deployments:
             return (None, True)
         windows: Final = tuple(
             window for deployment in deployments if (window := self._deployment_window(group, deployment)) is not None
@@ -3531,6 +3555,7 @@ class ComplexityRouter(CustomLogger):
                     ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs
                 )
             fallback_tier: Final = None if default_model_first else ComplexityTier.MEDIUM
+            default_tier_params: Final = self._litellm_params_for_model(fallback_tier, routed_model)
             return PreRoutingHookResponse(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
@@ -3539,7 +3564,9 @@ class ComplexityRouter(CustomLogger):
                     cause="default_fallback",
                     tier=fallback_tier,
                     conversation_continuing=conversation_continuing,
+                    tier_litellm_params=default_tier_params,
                 ),
+                litellm_params=default_tier_params,
             )
 
         ask: Final = user_message or ""
@@ -3566,6 +3593,7 @@ class ComplexityRouter(CustomLogger):
                 _tier_name(plan_floor),
                 routed_model,
             )
+            plan_tier_params: Final = self._litellm_params_for_model(plan_floor, routed_model)
             return PreRoutingHookResponse(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
@@ -3577,7 +3605,9 @@ class ComplexityRouter(CustomLogger):
                     matched_keyword=plan_mode_sentinel,
                     escalation_keyword=escalation_keyword,
                     escalated=False,
+                    tier_litellm_params=plan_tier_params,
                 ),
+                litellm_params=plan_tier_params,
             )
 
         override: Final = await self._resolve_keyword_tier_override(ask, request_kwargs)
@@ -3670,6 +3700,7 @@ class ComplexityRouter(CustomLogger):
                 outcome.signals,
                 fallback_model,
             )
+            fallback_tier_params: Final = self._litellm_params_for_model(None, fallback_model)
             return PreRoutingHookResponse(
                 model=fallback_model,
                 messages=messages if has_original_messages else None,
@@ -3680,7 +3711,9 @@ class ComplexityRouter(CustomLogger):
                     signals=outcome.signals,
                     escalation_keyword=escalation_keyword,
                     escalated=False,
+                    tier_litellm_params=fallback_tier_params,
                 ),
+                litellm_params=fallback_tier_params,
             )
         if self.config.adaptive:
             # hard_floor rather than a hard pick, and passed whenever the sentinel is present
diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py
index 37375d9727d..58f56d746c5 100644
--- a/litellm/router_strategy/complexity_router/config.py
+++ b/litellm/router_strategy/complexity_router/config.py
@@ -1089,6 +1089,20 @@ class ComplexityRouterConfig(BaseModel):
             "wording the built-ins don't cover, or after a client release changes its strings."
         ),
     )
+    max_tokens_from_tier_model: bool = Field(
+        default=True,
+        description=(
+            "Set max_tokens on every routed request to the output ceiling of the tier model it "
+            "lands on, replacing whatever the caller sent. A caller behind an auto-router cannot "
+            "pick one value that fits every tier: the smallest tier's ceiling starves a bigger "
+            "tier's thinking budget, and a bigger tier's ceiling is rejected by the smallest. The "
+            "ceiling is the smallest max_output_tokens across the tier model's deployments, read "
+            "from each deployment's model_info and then the model cost map; a tier model with a "
+            "deployment whose ceiling is unknown keeps the caller's value. A max_tokens, "
+            "max_completion_tokens or max_output_tokens in the tier's own litellm_params still "
+            "wins. Set false to forward the caller's value unchanged."
+        ),
+    )
     route_housekeeping_to_cheapest_tier: bool = Field(
         default=True,
         description=(
diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
index 7070617ce3e..74799072984 100644
--- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
+++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
@@ -7272,7 +7272,12 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth:
     )
 
 
-_PLANTED_STAMPS = {"attempted_fallbacks": 99, "original_model_group": "spoofed-group", "client_key": "client_value"}
+_PLANTED_STAMPS = {
+    "attempted_fallbacks": 99,
+    "original_model_group": "spoofed-group",
+    "_client_output_ceiling": {"api_base": "https://attacker.example"},
+    "client_key": "client_value",
+}
 
 
 @pytest.mark.asyncio
@@ -7301,6 +7306,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
     assert "litellm_metadata" not in updated
     assert "attempted_fallbacks" not in updated["metadata"]
     assert "original_model_group" not in updated["metadata"]
+    assert "_client_output_ceiling" not in updated["metadata"]
     assert updated["metadata"]["client_key"] == "client_value"
 
 
diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py
index b0f2a65bb9b..565e77b0ae0 100644
--- a/tests/test_litellm/router_strategy/test_complexity_router.py
+++ b/tests/test_litellm/router_strategy/test_complexity_router.py
@@ -16,6 +16,7 @@ from pydantic import ValidationError
 
 import litellm
 from litellm import Router
+from litellm.integrations.custom_logger import CustomLogger
 from litellm.router_utils.auto_router_model_naming import (
     CUSTOMIZATION_CAPABILITY,
     GATED_AUTO_ROUTER_CAPABILITIES,
@@ -24,7 +25,12 @@ from litellm.router_utils.auto_router_model_naming import (
 )
 from litellm._logging import verbose_router_logger
 from litellm.caching.dual_cache import DualCache
-from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
+from litellm.constants import (
+    OUTPUT_TOKEN_CEILING_PARAMS,
+    RETURN_RAW_MODEL_NAME_METADATA_KEY,
+    SESSION_ID_GENERATED_METADATA_KEY,
+)
+from litellm.router import as_output_cap
 from litellm.router_strategy.complexity_router.complexity_router import (
     _CLASSIFICATION_CURRENT_MESSAGE_ONLY,
     _CLASSIFICATION_WITH_CONVERSATION,
@@ -3415,11 +3421,11 @@ class TestRouterPreRoutingAliasOverrides:
 
     def test_drop_client_effort_carriers_helper_edge_shapes(self):
         no_pin: Dict = {"thinking": {"type": "adaptive"}}
-        Router._drop_client_effort_carriers_a_tier_pin_supersedes(no_pin, {"temperature": 0.1})
+        Router._drop_client_carriers_a_tier_pin_supersedes(no_pin, {"temperature": 0.1})
         assert no_pin == {"thinking": {"type": "adaptive"}}
 
         non_dict_carriers: Dict = {"output_config": "max", "reasoning": 3}
-        Router._drop_client_effort_carriers_a_tier_pin_supersedes(non_dict_carriers, {"reasoning_effort": "low"})
+        Router._drop_client_carriers_a_tier_pin_supersedes(non_dict_carriers, {"reasoning_effort": "low"})
         assert non_dict_carriers == {"output_config": "max", "reasoning": 3}
 
         effort_only: Dict = {"output_config": {"effort": "max"}, "reasoning": {"effort": "high"}}
@@ -3873,11 +3879,11 @@ class TestRouterPreRoutingSharedAliasName:
         }
 
     @staticmethod
-    async def _routed_call_kwargs(router: Router, **request_params) -> dict:
+    async def _routed_call_kwargs(router: Router, prompt: str = "hi", **request_params) -> dict:
         mock_acompletion = AsyncMock(return_value=litellm.ModelResponse(choices=[{"message": {"content": "hi"}}]))
         with patch.object(litellm, "acompletion", mock_acompletion):
             await router.acompletion(
-                model="smart-router", messages=[{"role": "user", "content": "hi"}], **request_params
+                model="smart-router", messages=[{"role": "user", "content": prompt}], **request_params
             )
         return mock_acompletion.call_args.kwargs
 
@@ -13278,3 +13284,379 @@ class TestClassifierVision:
     def test_max_images_must_be_positive(self):
         with pytest.raises(ValidationError):
             ClassifierLLMConfig(model="clf", vision={"enabled": True, "max_images": 0})
+
+
+class TestMaxTokensFromTierModel:
+    """The auto-router replaces the caller's output ceiling with the tier model's own, so one
+    client-side value no longer starves a bigger tier or gets rejected by a smaller one."""
+
+    COMPLEX_PROMPT: Final = (
+        "Design a distributed rate limiter with Redis, sharding and failover. Analyze the consistency "
+        "tradeoffs and implement the algorithm step by step with tests."
+    )
+    SMALL: Final = {
+        "model_name": "small",
+        "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "k"},
+        "model_info": {"max_output_tokens": 8192},
+    }
+
+    @staticmethod
+    def _router(
+        tier_litellm_params: dict | None = None,
+        max_tokens_from_tier_model: bool | None = None,
+        simple_deployments: list[dict] | None = None,
+        extra_config: dict | None = None,
+    ) -> Router:
+        simple_tier: dict = {"model_name": "small"}
+        if tier_litellm_params:
+            simple_tier["litellm_params"] = tier_litellm_params
+        config: dict = {
+            "tiers": {"SIMPLE": simple_tier, "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"},
+            **(extra_config or {}),
+        }
+        if max_tokens_from_tier_model is not None:
+            config["max_tokens_from_tier_model"] = max_tokens_from_tier_model
+        return Router(
+            model_list=[
+                {
+                    "model_name": "smart-router",
+                    "litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": config},
+                },
+                *(simple_deployments or [TestMaxTokensFromTierModel.SMALL]),
+                {
+                    "model_name": "big",
+                    "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "k"},
+                    "model_info": {"max_output_tokens": 64000},
+                },
+            ]
+        )
+
+    @staticmethod
+    async def _routed(router: Router, prompt: str = "hi", **request_kwargs) -> dict:
+        """Drive the real routing entry point and return the request kwargs it leaves behind."""
+        deployment = await router.async_get_available_deployment(
+            model="smart-router", request_kwargs=request_kwargs, messages=[{"role": "user", "content": prompt}]
+        )
+        return {"model": deployment["litellm_params"]["model"], **request_kwargs}
+
+    @staticmethod
+    async def _routed_responses(router: Router, prompt: str = "hi", **request_kwargs) -> dict:
+        """The Responses surface hands the router `input` both as the prompt argument and inside the
+        request kwargs, so the hook sees the same shape the real call carries."""
+        routed: dict = {"input": prompt, **request_kwargs}
+        deployment = await router.async_get_available_deployment(
+            model="smart-router", request_kwargs=routed, input=prompt
+        )
+        return {"model": deployment["litellm_params"]["model"], **routed}
+
+    @pytest.mark.asyncio
+    async def test_client_ceiling_is_replaced_by_the_routed_tier_models_ceiling(self):
+        router = self._router()
+
+        simple = await self._routed(router, max_tokens=8192)
+        complex_ = await self._routed(router, self.COMPLEX_PROMPT, max_tokens=8192)
+
+        assert (simple["model"], simple["max_tokens"]) == ("anthropic/claude-haiku-4-5", 8192)
+        assert (complex_["model"], complex_["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+        assert "max_output_tokens" not in complex_
+
+    @pytest.mark.asyncio
+    async def test_every_client_carrier_of_the_ceiling_is_replaced(self):
+        sent = await self._routed(self._router(), self.COMPLEX_PROMPT, max_completion_tokens=8192)
+
+        assert sent["max_tokens"] == 64000
+        assert "max_completion_tokens" not in sent
+
+    @pytest.mark.asyncio
+    async def test_responses_surface_gets_the_ceiling_under_its_own_name(self):
+        sent = await self._routed_responses(self._router(), self.COMPLEX_PROMPT, max_output_tokens=8192)
+
+        assert (sent["model"], sent["max_output_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+        assert "max_tokens" not in sent
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "tier_params, responses_call",
+        [
+            ({"max_tokens": 4321}, False),
+            ({"max_tokens": 4321}, True),
+            ({"max_completion_tokens": 4321}, False),
+            ({"max_completion_tokens": 4321}, True),
+            ({"max_output_tokens": 4321}, False),
+        ],
+    )
+    async def test_operators_own_tier_ceiling_wins_under_the_surface_name(self, tier_params, responses_call):
+        router = self._router(tier_litellm_params=tier_params)
+        if responses_call:
+            sent = await self._routed_responses(router, max_output_tokens=8192)
+        else:
+            sent = await self._routed(router, max_tokens=8192)
+
+        surface_key = "max_output_tokens" if responses_call else "max_tokens"
+        assert sent[surface_key] == 4321
+        assert not (OUTPUT_TOKEN_CEILING_PARAMS - {surface_key}) & sent.keys()
+
+    @pytest.mark.asyncio
+    async def test_opting_out_forwards_the_client_value_unchanged(self):
+        sent = await self._routed(self._router(max_tokens_from_tier_model=False), self.COMPLEX_PROMPT, max_tokens=8192)
+
+        assert sent["max_tokens"] == 8192
+
+    @pytest.mark.asyncio
+    async def test_a_tier_model_with_an_unknown_ceiling_keeps_the_client_value(self):
+        unmapped: dict = {"model_name": "small", "litellm_params": {"model": "openai/not-in-any-map", "api_key": "k"}}
+
+        sent = await self._routed(self._router(simple_deployments=[self.SMALL, unmapped]), max_tokens=4000)
+
+        assert sent["max_tokens"] == 4000
+
+    @pytest.mark.asyncio
+    async def test_a_multi_deployment_tier_model_uses_its_smallest_ceiling(self):
+        smaller: dict = {
+            **self.SMALL,
+            "litellm_params": {**self.SMALL["litellm_params"], "api_key": "k2"},
+            "model_info": {"max_output_tokens": 4096},
+        }
+
+        sent = await self._routed(self._router(simple_deployments=[self.SMALL, smaller]), max_tokens=100000)
+
+        assert sent["max_tokens"] == 4096
+
+    @pytest.mark.asyncio
+    async def test_ceiling_falls_back_to_the_cost_map(self, monkeypatch):
+        monkeypatch.setitem(
+            litellm.model_cost,
+            "auto-cap-probe-model",
+            {"litellm_provider": "openai", "mode": "chat", "max_output_tokens": 4242, "max_input_tokens": 100000},
+        )
+        mapped_only: dict = {
+            "model_name": "small",
+            "litellm_params": {"model": "openai/auto-cap-probe-model", "api_key": "k"},
+        }
+
+        sent = await self._routed(self._router(simple_deployments=[mapped_only]), max_tokens=8192)
+
+        assert sent["max_tokens"] == 4242
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("client_kwargs", [{}, {"max_tokens": 0}], ids=["omitted", "zero"])
+    async def test_omitted_and_zero_are_replaced_like_any_other_value(self, client_kwargs):
+        sent = await self._routed(self._router(), self.COMPLEX_PROMPT, **client_kwargs)
+
+        assert sent["max_tokens"] == 64000
+
+    @pytest.mark.parametrize(
+        "tier_params, responses_call, expected",
+        [
+            ({"max_tokens": 1, "temperature": 0.2}, False, {"max_tokens": 1, "temperature": 0.2}),
+            ({"max_tokens": 1}, True, {"max_output_tokens": 1}),
+            ({"max_completion_tokens": 2}, False, {"max_tokens": 2}),
+            ({"max_completion_tokens": 2}, True, {"max_output_tokens": 2}),
+            ({"max_output_tokens": 3}, False, {"max_tokens": 3}),
+            ({"max_output_tokens": 3}, True, {"max_output_tokens": 3}),
+            ({"max_tokens": 1, "max_completion_tokens": 2, "max_output_tokens": 3}, False, {"max_tokens": 1}),
+            ({"max_tokens": 1, "max_completion_tokens": 2, "max_output_tokens": 3}, True, {"max_output_tokens": 3}),
+            ({"max_completion_tokens": 2, "max_output_tokens": 3}, False, {"max_tokens": 2}),
+            ({"reasoning_effort": "low"}, True, {"reasoning_effort": "low"}),
+        ],
+    )
+    def test_every_tier_alias_collapses_onto_the_surface_key(self, tier_params, responses_call, expected):
+        assert dict(Router._tier_ceiling_under_the_surface_name(tier_params, responses_call=responses_call)) == expected
+
+    @pytest.mark.asyncio
+    async def test_the_default_fallback_exit_carries_the_ceiling(self):
+        routed: dict = {"max_tokens": 8192}
+        deployment = await self._router().async_get_available_deployment(
+            model="smart-router", request_kwargs=routed, messages=[{"role": "system", "content": "be nice"}]
+        )
+
+        assert routed["metadata"]["routing_decision"]["cause"] == "default_fallback"
+        assert (deployment["litellm_params"]["model"], routed["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+
+    @pytest.mark.asyncio
+    async def test_the_plan_mode_exit_carries_the_ceiling(self):
+        routed: dict = {"max_tokens": 8192}
+        deployment = await self._router(
+            extra_config={"plan_mode_min_tier": "REASONING"}
+        ).async_get_available_deployment(
+            model="smart-router",
+            request_kwargs=routed,
+            messages=[
+                {"role": "user", "content": "plan the refactor"},
+                {"role": "system", "content": "Plan mode is active"},
+            ],
+        )
+
+        assert routed["metadata"]["routing_decision"]["cause"] == "plan_mode"
+        assert (deployment["litellm_params"]["model"], routed["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+
+    @pytest.mark.asyncio
+    async def test_a_default_model_landing_with_no_tier_still_gets_its_ceiling(self):
+        strategy = ComplexityRouter(
+            model_name="smart-router",
+            litellm_router_instance=self._router(),
+            complexity_router_config={"tiers": {"SIMPLE": "small"}, "default_model": "big"},
+        )
+
+        assert dict(strategy._litellm_params_for_model(None, "big")) == {"max_tokens": 64000}
+
+    @pytest.mark.asyncio
+    async def test_a_fallback_into_a_plain_group_gets_the_callers_ceiling_back(self):
+        """A model-group fallback re-enters routing with the same kwargs; a Sonnet-sized ceiling
+        must not ride onto the plain group the caller configured as the fallback."""
+        big: dict = {
+            "model_name": "big",
+            "litellm_params": {
+                "model": "anthropic/claude-sonnet-5",
+                "api_key": "k",
+                "mock_response": "litellm.InternalServerError",
+            },
+            "model_info": {"max_output_tokens": 64000},
+        }
+        plain: dict = {
+            "model_name": "plain",
+            "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "k", "mock_response": "ok"},
+        }
+        router = Router(
+            model_list=[
+                {
+                    "model_name": "smart-router",
+                    "litellm_params": {
+                        "model": "auto_router/complexity_router",
+                        "complexity_router_config": {
+                            "tiers": {"SIMPLE": "big", "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"}
+                        },
+                    },
+                },
+                big,
+                plain,
+            ],
+            fallbacks=[{"smart-router": ["plain"]}],
+            num_retries=0,
+        )
+        recorder = _OutputCeilingRecorder()
+        litellm.callbacks.append(recorder)
+        try:
+            await router.acompletion(
+                model="smart-router", messages=[{"role": "user", "content": self.COMPLEX_PROMPT}], max_tokens=8192
+            )
+        finally:
+            litellm.callbacks.remove(recorder)
+
+        assert recorder.seen == [("claude-sonnet-5", 64000), ("claude-haiku-4-5", 8192)]
+
+    @pytest.mark.asyncio
+    async def test_a_caller_seeded_stamp_cannot_inject_kwargs_on_a_plain_group(self):
+        """The stamp sits in a metadata bucket a caller can write; a planted one must yield
+        nothing but integer ceiling carriers, never a redirected api_base or credential."""
+        planted: dict = {
+            "api_base": "https://attacker.example",
+            "api_key": "stolen",
+            "max_tokens": "not-an-int",
+            "max_completion_tokens": True,
+            "max_output_tokens": 321,
+        }
+        routed: dict = {"max_tokens": 8192, "metadata": {"_client_output_ceiling": planted}}
+
+        await self._router().async_get_available_deployment(
+            model="big", request_kwargs=routed, messages=[{"role": "user", "content": "hi"}]
+        )
+
+        assert {k: v for k, v in routed.items() if k not in ("metadata", "model_info")} == {"max_output_tokens": 321}
+
+    @pytest.mark.asyncio
+    async def test_the_pass_through_routing_entry_point_pins_and_restores_the_same_way(self):
+        pass_through: dict = {**self.SMALL["litellm_params"], "use_in_pass_through": True}
+        small: dict = {**self.SMALL, "litellm_params": pass_through}
+        plain: dict = {**small, "model_name": "plain"}
+        router = self._router(simple_deployments=[small, plain])
+        for deployment in router.model_list:
+            deployment["litellm_params"]["use_in_pass_through"] = True
+        routed: dict = {"max_tokens": 8192}
+
+        deployment = await router.async_get_available_deployment_for_pass_through(
+            model="smart-router", request_kwargs=routed, messages=[{"role": "user", "content": self.COMPLEX_PROMPT}]
+        )
+        pinned = routed["max_tokens"]
+        await router.async_get_available_deployment_for_pass_through(
+            model="plain", request_kwargs=routed, messages=[{"role": "user", "content": "hi"}]
+        )
+
+        assert (deployment["litellm_params"]["model"], pinned, routed["max_tokens"]) == (
+            "anthropic/claude-sonnet-5",
+            64000,
+            8192,
+        )
+
+    @pytest.mark.asyncio
+    async def test_the_classifier_fallback_exit_carries_the_ceiling(self):
+        router = self._router(
+            extra_config={
+                "classifier_type": "llm",
+                "classifier_llm_config": {"model": "no-such-classifier", "timeout_ms": 400},
+                "classifier_fallback": "default_model",
+                "default_model": "big",
+            }
+        )
+        routed: dict = {"max_tokens": 8192}
+
+        deployment = await router.async_get_available_deployment(
+            model="smart-router", request_kwargs=routed, messages=[{"role": "user", "content": "hi"}]
+        )
+
+        assert routed["metadata"]["routing_decision"]["cause"] == "default_model_fallback"
+        assert (deployment["litellm_params"]["model"], routed["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+
+    @pytest.mark.parametrize(
+        "value, expected",
+        [(8192, 8192), ("8192", 8192), (100.9, 100), (0, 0), (-1, None), (True, None), ("x", None), (None, None)],
+    )
+    def test_a_client_cap_is_read_as_an_integer_or_ignored(self, value, expected):
+        assert as_output_cap(value) == expected
+
+    def test_restoring_the_callers_ceiling_reads_the_stamp_and_replaces_every_carrier(self):
+        stamped: dict = {"max_output_tokens": 500, "metadata": {"_client_output_ceiling": {"max_tokens": 8192}}}
+        Router._restore_client_ceiling_no_tier_pins(stamped)
+        assert {k: v for k, v in stamped.items() if k != "metadata"} == {"max_tokens": 8192}
+
+        coerced: dict = {
+            "max_tokens": 64000,
+            "metadata": {"_client_output_ceiling": {"max_tokens": "8192", "max_completion_tokens": 100.0}},
+        }
+        Router._restore_client_ceiling_no_tier_pins(coerced)
+        assert {k: v for k, v in coerced.items() if k != "metadata"} == {
+            "max_tokens": 8192,
+            "max_completion_tokens": 100,
+        }
+
+        unstamped: dict = {"max_tokens": 64000, "metadata": {}}
+        Router._restore_client_ceiling_no_tier_pins(unstamped)
+        assert unstamped["max_tokens"] == 64000
+
+    @pytest.mark.asyncio
+    async def test_pinning_stamps_the_callers_carriers_once(self):
+        router = self._router()
+        request_kwargs: dict = {"max_completion_tokens": 8192}
+
+        first = router._pin_tier_params_onto_request(
+            model="big", tier_litellm_params={"max_tokens": 64000}, request_kwargs=request_kwargs, responses_call=False
+        )
+        second = router._pin_tier_params_onto_request(
+            model="big", tier_litellm_params={"max_tokens": 32000}, request_kwargs=request_kwargs, responses_call=False
+        )
+        none = router._pin_tier_params_onto_request(
+            model="big", tier_litellm_params=None, request_kwargs=request_kwargs, responses_call=False
+        )
+
+        assert (first, second, none) == (True, True, False)
+        assert request_kwargs["max_tokens"] == 32000
+        assert request_kwargs["metadata"]["_client_output_ceiling"] == {"max_completion_tokens": 8192}
+
+
+class _OutputCeilingRecorder(CustomLogger):
+    def __init__(self) -> None:
+        super().__init__()
+        self.seen: list[tuple[str, int | None]] = []
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        self.seen.append((model, kwargs.get("optional_params", {}).get("max_tokens")))
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index d1431d8956f..b52f0922390 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -34996,6 +34996,12 @@ export interface components {
              * @default 0.5
              */
             match_threshold: number;
+            /**
+             * Max Tokens From Tier Model
+             * @description Set max_tokens on every routed request to the output ceiling of the tier model it lands on, replacing whatever the caller sent. A caller behind an auto-router cannot pick one value that fits every tier: the smallest tier's ceiling starves a bigger tier's thinking budget, and a bigger tier's ceiling is rejected by the smallest. The ceiling is the smallest max_output_tokens across the tier model's deployments, read from each deployment's model_info and then the model cost map; a tier model with a deployment whose ceiling is unknown keeps the caller's value. A max_tokens, max_completion_tokens or max_output_tokens in the tier's own litellm_params still wins. Set false to forward the caller's value unchanged.
+             * @default true
+             */
+            max_tokens_from_tier_model: boolean;
             /**
              * Modality Pin Override
              * @description Let modality_routing replace a kept session-affinity pin on the turns that carry an image. Without this, a session pinned to a text-only model fails every image turn with a provider 400, since the pin is exempt from the modality gate. When enabled, such a turn routes to a capable model for that request only and the stored pin is left untouched, so the next text turn replays the session's own model; the override is reported as cause modality_pin_override and is never itself pinned. Inert unless modality_routing is also enabled.