Merge litellm_internal_staging: reconcile classifier prompt override (#35855) with custom tiers

#35855 lets classifier_llm_config.system_prompt replace the classifier's entire
system role, with its own classifier_fallback='default_model' escape hatch since
the heuristic cannot score a repurposed taxonomy. This branch's
classification_prompt replaces only the rubric's opening instructions and always
keeps the per-tier bullets and the trust boundary. Both knobs now live on the
unified builder: custom_prompt returns verbatim, preamble composes with the
active tier entries, and the exception path prefers a custom set's fallback_tier
before consulting classifier_fallback. Three combinations are rejected at config
write because each pair claims the same seat: system_prompt with
classification_prompt, system_prompt with tier_definitions, and
classifier_fallback='default_model' with tier_definitions. The default-prompt
endpoint keeps serving labeled rubrics through the new canonical_rubric_entries
helper, and RoutingDecisionCause carries both branches' new causes.

.github/workflows/test-linting.yml

@@ -23,10 +23,21 @@ jobs:
         # Any-discipline) would otherwise blame on this branch.
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
+          fetch-depth: 1
           clean: true
           persist-credentials: false
 
+      - name: Fetch gate base (merge-base with target branch)
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          MERGE_BASE=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$MERGE_BASE"
+          git fetch --no-tags --depth=1 origin "$MERGE_BASE"
+          echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -60,10 +71,8 @@ jobs:
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Check ruff format
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          git diff --name-only --diff-filter=ACMR "$BASE_SHA"...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
+          git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
           if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
             echo "No changed litellm Python files to check with ruff format."
             exit 0
@@ -86,32 +95,24 @@ jobs:
           cd ..
 
       - name: Check strict-rule budget (delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/ruff_strict_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/type_discipline_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
 
       - name: Print OpenAI version
         run: |
           uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
 
       - name: Check basedpyright budget (delta vs base)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          uv run --no-sync python scripts/type_check_gate.py --base "$BASE_SHA"
+          uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check tests/e2e basedpyright (zero errors)
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          if git diff --name-only --diff-filter=ACMRD "$BASE_SHA"...HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
+          if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
             uv run --no-sync basedpyright tests/e2e
           else
             echo "No changed tests/e2e Python files; skipping."
@@ -140,9 +141,15 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          fetch-depth: 0
+          fetch-depth: 1
           persist-credentials: false
 
+      - name: Fetch ratchet base
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          git fetch --no-tags --depth=1 origin "$BASE_SHA"
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:

backend/routes/allowlist.py

@@ -82,6 +82,9 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/user_agent",
     "/usage/",
     "/daily/",
+    # Deployment-wide gateway request counts. Scoped to the analytics read rather
+    # than all of /gateway/, which stays free for data-plane routes.
+    "/gateway/daily/",
     # CloudZero cost-export admin (init / settings / export / dry-run / delete)
     "/cloudzero/",
     # Caching admin

basedpyright-code-budget.json

@@ -1,6 +1,6 @@
 {
   "reportAny": {
-    "limit": 29809
+    "limit": 29806
   },
   "reportArgumentType": {
     "limit": 2645
@@ -21,10 +21,10 @@
     "limit": 215
   },
   "reportDuplicateImport": {
-    "limit": 24
+    "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 9473
+    "limit": 9469
   },
   "reportFunctionMemberAccess": {
     "limit": 7
@@ -105,13 +105,13 @@
     "limit": 113
   },
   "reportUnknownMemberType": {
-    "limit": 40452
+    "limit": 40447
   },
   "reportUnknownParameterType": {
     "limit": 20309
   },
   "reportUnknownVariableType": {
-    "limit": 31978
+    "limit": 31879
   },
   "reportUnnecessaryCast": {
     "limit": 124
@@ -126,7 +126,7 @@
     "limit": 866
   },
   "reportUntypedBaseClass": {
-    "limit": 165
+    "limit": 72
   },
   "reportUntypedFunctionDecorator": {
     "limit": 33
@@ -138,9 +138,9 @@
     "limit": 139
   },
   "reportUnusedImport": {
-    "limit": 588
+    "limit": 556
   },
   "reportUnusedVariable": {
-    "limit": 147
+    "limit": 146
   }
 }

docker/build_from_pip/Dockerfile.build_from_pip

@@ -47,7 +47,13 @@ RUN uv venv --python python && \
       "prisma==0.11.0" \
       "openai==2.24.0"
 
-RUN prisma generate --schema=./schema.prisma
+RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
+    npm_config_cache=/root/.npm \
+    prisma generate --schema=./schema.prisma && \
+    chmod -R a+rX /opt/prisma && \
+    python -c "import sys; from prisma.client import BINARY_PATHS; bad = sorted(p for group in BINARY_PATHS.model_dump().values() for p in group.values() if not p.startswith('/opt/prisma/')); sys.exit('prisma engines baked outside /opt/prisma: %r' % bad) if bad else None"
+
+ENV PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
 
 EXPOSE 4000/tcp
 

litellm-proxy-extras/litellm_proxy_extras/migrations/20260803000000_add_daily_gateway_requests/migration.sql

@@ -0,0 +1,15 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_DailyGatewayRequests" (
+    "date" TEXT NOT NULL,
+    "category" TEXT NOT NULL,
+    "route" TEXT NOT NULL,
+    "successful_requests" BIGINT NOT NULL DEFAULT 0,
+    "failed_requests" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_DailyGatewayRequests_pkey" PRIMARY KEY ("date","category","route")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_DailyGatewayRequests_date_idx" ON "LiteLLM_DailyGatewayRequests"("date");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260805000000_add_autorouter_session_rollup/migration.sql

@@ -0,0 +1,31 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterSession" (
+    "api_key" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "router_type" TEXT NOT NULL,
+    "first_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_turn_at" TIMESTAMP(3) NOT NULL,
+    "last_model" TEXT NOT NULL,
+    "models" JSONB NOT NULL DEFAULT '{}',
+    "turns" INTEGER NOT NULL DEFAULT 0,
+    "unordered_turns" INTEGER NOT NULL DEFAULT 0,
+    "covered_turns" INTEGER NOT NULL DEFAULT 0,
+    "cache_hits" INTEGER NOT NULL DEFAULT 0,
+    "same_model_turns" INTEGER NOT NULL DEFAULT 0,
+    "same_model_hits" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_turns" INTEGER NOT NULL DEFAULT 0,
+    "first_visit_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_turns" INTEGER NOT NULL DEFAULT 0,
+    "return_hits" INTEGER NOT NULL DEFAULT 0,
+    "return_expired_misses" INTEGER NOT NULL DEFAULT 0,
+    "return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0,
+    "ttl_5m_turns" INTEGER NOT NULL DEFAULT 0,
+    "ttl_1h_turns" INTEGER NOT NULL DEFAULT 0,
+    "total_tokens" BIGINT NOT NULL DEFAULT 0,
+    "spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
+
+    CONSTRAINT "LiteLLM_AutoRouterSession_pkey" PRIMARY KEY ("api_key", "session_id", "router_name")
+);
+
+CREATE INDEX IF NOT EXISTS "idx_autorouter_session_last_turn" ON "LiteLLM_AutoRouterSession"("last_turn_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1118,6 +1118,26 @@ model LiteLLM_DailyToolSpend {
   @@id([date, tool_name])
 }
 
+// Gateway request counts recorded at the ASGI edge by
+// BillableRequestMetricsMiddleware. This is the source of truth for SGR
+// (successful gateway requests): it counts what the proxy actually answered,
+// independent of whether the request reached litellm's logging callbacks.
+// The key carries no deployment or caller dimension. Every part of it is
+// chosen by the proxy and drawn from a closed set, so the table is bounded by
+// (days x categories x routes) rather than by anything a caller can vary.
+model LiteLLM_DailyGatewayRequests {
+  date                String
+  category            String
+  route               String
+  successful_requests BigInt   @default(0)
+  failed_requests     BigInt   @default(0)
+  created_at          DateTime @default(now())
+  updated_at          DateTime @updatedAt
+
+  @@id([date, category, route])
+  @@index([date])
+}
+
 // Prompt table for storing prompt configurations
 model LiteLLM_PromptTable {
   id String @id @default(uuid())
@@ -1393,6 +1413,37 @@ model LiteLLM_AdaptiveRouterSession {
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
 
+model LiteLLM_AutoRouterSession {
+  api_key               String
+  session_id            String
+  router_name           String
+  router_type           String
+  first_turn_at         DateTime
+  last_turn_at          DateTime
+  last_model            String
+  models                Json     @default("{}")
+  turns                 Int      @default(0)
+  unordered_turns       Int      @default(0)
+  covered_turns         Int      @default(0)
+  cache_hits            Int      @default(0)
+  same_model_turns      Int      @default(0)
+  same_model_hits       Int      @default(0)
+  first_visit_turns     Int      @default(0)
+  first_visit_hits      Int      @default(0)
+  return_turns          Int      @default(0)
+  return_hits           Int      @default(0)
+  return_expired_misses Int      @default(0)
+  return_within_ttl_misses  Int      @default(0)
+  ttl_5m_turns          Int      @default(0)
+  ttl_1h_turns          Int      @default(0)
+  total_tokens          BigInt   @default(0)
+  spend                 Float    @default(0)
+  saved_spend           Float    @default(0)
+
+  @@id([api_key, session_id, router_name])
+  @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //

litellm/_lazy_imports_registry.py

@@ -1461,32 +1461,30 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
 
 # Export all name tuples and import maps for use in _lazy_imports.py
 __all__ = [
-    # Name tuples
-    "COST_CALCULATOR_NAMES",
-    "LITELLM_LOGGING_NAMES",
-    "UTILS_NAMES",
-    "TOKEN_COUNTER_NAMES",
-    "LLM_CLIENT_CACHE_NAMES",
     "BEDROCK_TYPES_NAMES",
-    "TYPES_UTILS_NAMES",
     "CACHING_NAMES",
-    "HTTP_HANDLER_NAMES",
+    "COST_CALCULATOR_NAMES",
     "DOTPROMPT_NAMES",
+    "HTTP_HANDLER_NAMES",
+    "LITELLM_LOGGING_NAMES",
+    "LLM_CLIENT_CACHE_NAMES",
     "LLM_CONFIG_NAMES",
-    "TYPES_NAMES",
     "LLM_PROVIDER_LOGIC_NAMES",
+    "TOKEN_COUNTER_NAMES",
+    "TYPES_NAMES",
+    "TYPES_UTILS_NAMES",
     "UTILS_MODULE_NAMES",
-    # Import maps
-    "_UTILS_IMPORT_MAP",
-    "_COST_CALCULATOR_IMPORT_MAP",
-    "_TYPES_UTILS_IMPORT_MAP",
-    "_TOKEN_COUNTER_IMPORT_MAP",
+    "UTILS_NAMES",
     "_BEDROCK_TYPES_IMPORT_MAP",
     "_CACHING_IMPORT_MAP",
-    "_LITELLM_LOGGING_IMPORT_MAP",
+    "_COST_CALCULATOR_IMPORT_MAP",
     "_DOTPROMPT_IMPORT_MAP",
-    "_TYPES_IMPORT_MAP",
+    "_LITELLM_LOGGING_IMPORT_MAP",
     "_LLM_CONFIGS_IMPORT_MAP",
     "_LLM_PROVIDER_LOGIC_IMPORT_MAP",
+    "_TOKEN_COUNTER_IMPORT_MAP",
+    "_TYPES_IMPORT_MAP",
+    "_TYPES_UTILS_IMPORT_MAP",
+    "_UTILS_IMPORT_MAP",
     "_UTILS_MODULE_IMPORT_MAP",
 ]

litellm/_service_logger.py

@@ -1,6 +1,6 @@
 import asyncio
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 import litellm
 from litellm._logging import verbose_logger
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 
     from litellm.proxy._types import UserAPIKeyAuth
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     OTELClass = OpenTelemetry
 else:
     Span = Any

litellm/a2a_protocol/__init__.py

@@ -55,19 +55,15 @@ from litellm.a2a_protocol.main import (
 from litellm.types.agents import LiteLLMSendMessageResponse
 
 __all__ = [
-    # Client
-    "A2AClient",
-    # Functions
-    "asend_message",
-    "send_message",
-    "asend_message_streaming",
-    "aget_agent_card",
-    "create_a2a_client",
-    # Response types
-    "LiteLLMSendMessageResponse",
-    # Exceptions
-    "A2AError",
-    "A2AConnectionError",
     "A2AAgentCardError",
+    "A2AClient",
+    "A2AConnectionError",
+    "A2AError",
     "A2ALocalhostURLError",
+    "LiteLLMSendMessageResponse",
+    "aget_agent_card",
+    "asend_message",
+    "asend_message_streaming",
+    "create_a2a_client",
+    "send_message",
 ]

litellm/assistants/utils.py

@@ -8,8 +8,8 @@ from ..types.llms.openai import *
 
 def get_optional_params_add_message(
     role: str | None,
-    content: str | List[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
-    attachments: List[Attachment] | None,
+    content: str | list[MessageContentTextObject | MessageContentImageFileObject | MessageContentImageURLObject] | None,
+    attachments: list[Attachment] | None,
     metadata: dict | None,
     custom_llm_provider: str,
     **kwargs,
@@ -57,7 +57,7 @@ def get_optional_params_add_message(
         optional_params = litellm.AzureOpenAIAssistantsAPIConfig().map_openai_params_create_message_params(
             non_default_params=non_default_params, optional_params=optional_params
         )
-    for k in passed_params.keys():
+    for k in passed_params:
         if k not in default_params:
             optional_params[k] = passed_params[k]
     return optional_params
@@ -128,7 +128,7 @@ def get_optional_params_image_gen(
         if n is not None:
             optional_params["sampleCount"] = int(n)
 
-    for k in passed_params.keys():
+    for k in passed_params:
         if k not in default_params:
             optional_params[k] = passed_params[k]
     return optional_params

litellm/caching/base_cache.py

@@ -9,12 +9,12 @@ Has 4 methods:
 """
 
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/caching/disk_cache.py

@@ -1,12 +1,12 @@
 import json
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from .base_cache import BaseCache
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/caching/dual_cache.py

@@ -13,7 +13,7 @@ import time
 import traceback
 from concurrent.futures import ThreadPoolExecutor
 from threading import Lock
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 if TYPE_CHECKING:
     from litellm.types.caching import RedisPipelineIncrementOperation
@@ -29,7 +29,7 @@ from .redis_cache import RedisCache
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/caching/evicted_client_closer.py

@@ -0,0 +1,276 @@
+"""
+Deferred close of HTTP/SDK clients that the LLM client cache has evicted.
+
+Eviction only drops the cache's reference to a client. Every OpenAI/Azure SDK
+client is a reference cycle (each resource namespace holds the client back), so
+an evicted client and its pooled TCP connections survive until a generational
+collection runs, which under load is thousands of requests later.
+
+Closing at eviction time is not an option: a request that was handed the client
+just before it was evicted is still using it, and closing it underneath that
+request raises ``RuntimeError: Cannot send a request, as the client has been
+closed.``
+
+So an evicted client is closed once two conditions hold. A grace window must
+have passed since its eviction, which covers a request that holds the client
+but is momentarily not on the wire, and the client must report no connection in
+flight. The second condition is what keeps the first honest: a request may run
+for ``litellm.request_timeout`` seconds, 6000 by default, and a streaming
+response is bounded only by how long the upstream keeps sending, so no deadline
+on its own can promise that a request has finished.
+
+Only clients litellm itself created are closed; a client the caller supplied is
+left alone because litellm does not own its lifecycle.
+
+A client that closes synchronously is closed from wherever the cache is next
+used. One whose close is a coroutine needs the event loop it was evicted on, so
+it waits for a call from that loop rather than having work scheduled onto a loop
+it does not belong to. Queued clients are therefore bucketed by what it takes to
+close them, and each bucket is ordered by deadline, so a reap walks the entries
+that are due rather than the whole queue.
+
+The queue holds its clients weakly, so waiting out a grace window never keeps
+alive anything the collector would have reclaimed first.
+"""
+
+import asyncio
+import contextlib
+import inspect
+import threading
+import time
+import weakref
+from collections import deque
+from collections.abc import Awaitable, Callable, Iterator
+from dataclasses import dataclass, replace
+from typing import Final
+
+from litellm.constants import (
+    EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
+    EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
+)
+
+_CLOSABLE_ANYWHERE: Final = "closable-anywhere"
+_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
+
+_BucketKey = str | int
+
+
+@dataclass(frozen=True, slots=True)
+class _PendingClose:
+    """A queued close.
+
+    The client is held weakly, so queueing one never keeps alive anything the
+    collector would otherwise have reclaimed first.
+
+    ``needs_loop`` is set for a client whose close is a coroutine; those can only
+    be closed from the event loop they were evicted on, recorded in ``loop_id``.
+    A client that closes synchronously carries neither constraint.
+    """
+
+    client_ref: "weakref.ref[object]"
+    loop_id: int | None
+    needs_loop: bool
+    close_after: float
+
+
+def _bucket_key(pending: _PendingClose) -> _BucketKey:
+    """Which reaps can close this entry: any at all, any running a loop, or one loop's."""
+    if not pending.needs_loop:
+        return _CLOSABLE_ANYWHERE
+    if pending.loop_id is None:
+        return _CLOSABLE_ON_ANY_LOOP
+    return pending.loop_id
+
+
+def _running_loop_id() -> int | None:
+    try:
+        return id(asyncio.get_running_loop())
+    except RuntimeError:
+        return None
+
+
+def _close_function(client: object) -> Callable[[], object] | None:
+    close_fn: Final[Callable[[], object] | None] = getattr(client, "aclose", None) or getattr(client, "close", None)
+    return close_fn
+
+
+def _transport_of(client: object) -> object:
+    """The httpx transport behind an SDK wrapper, a litellm handler, or a bare client."""
+    for holder in (getattr(client, "_client", None), getattr(client, "client", None), client):
+        transport: object = getattr(holder, "_transport", None)
+        if transport is not None:
+            return transport
+    return None
+
+
+def _connection_is_idle(connection: object) -> bool:
+    """A pooled connection is idle unless it is servicing a request."""
+    is_idle: Final[object] = getattr(connection, "is_idle", None)
+    return bool(is_idle()) if callable(is_idle) else True
+
+
+def _pool_has_busy_connection(transport: object) -> bool | None:
+    """Whether the httpcore pool behind the transport is servicing a request.
+
+    ``None`` when there is no such pool, so the caller can ask the other backend.
+    """
+    pooled: Final[object] = getattr(getattr(transport, "_pool", None), "connections", None)
+    if not isinstance(pooled, (list, tuple)):
+        return None
+    return any(
+        not _connection_is_idle(connection)  # pyright: ignore[reportUnknownArgumentType]  # untyped pool list
+        for connection in pooled  # pyright: ignore[reportUnknownVariableType]  # untyped pool list
+    )
+
+
+def _has_connection_in_flight(client: object) -> bool:
+    """Whether the client is servicing a request right now.
+
+    Both connection backends litellm uses already account for the connections
+    they have handed out, so this reads the client's own lease accounting rather
+    than inferring it from elapsed time: httpcore reports a non-idle connection
+    for the whole of a response including a stream, and aiohttp holds the
+    connection in ``_acquired`` over the same span.
+
+    A client that cannot answer is reported as idle, which leaves the grace
+    window as the only guard, exactly as it was before this check existed.
+    """
+    try:
+        transport: Final = _transport_of(client)
+        pooled_busy: Final = _pool_has_busy_connection(transport)
+        if pooled_busy is not None:
+            return pooled_busy
+        session: Final[object] = getattr(transport, "client", None)
+        return bool(getattr(getattr(session, "connector", None), "_acquired", None))
+    except Exception:  # noqa: BLE001 - a client that cannot report its state is treated as idle
+        return False
+
+
+async def _close_quietly(closing: Awaitable[object]) -> None:
+    with contextlib.suppress(Exception):
+        await closing
+
+
+class EvictedClientCloser:
+    """Closes evicted, litellm-owned clients once they are idle and out of grace."""
+
+    def __init__(
+        self,
+        grace_seconds: float = EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS,
+        max_pending: int = EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self._grace_seconds = grace_seconds
+        self._max_pending = max_pending
+        self._clock = clock
+        self._owned: weakref.WeakSet[object] = weakref.WeakSet()
+        self._buckets: dict[_BucketKey, deque[_PendingClose]] = {}  # mutable-ok: deadline-ordered queues
+        self._pending_count = 0
+        self._queue_lock = threading.Lock()  # the cache is reachable from every worker thread's loop
+        self._close_tasks: set[asyncio.Task[None]] = set()  # mutable-ok: strong refs to running closes
+
+    def mark_owned(self, client: object) -> None:
+        """Record that litellm created this client, so it may be closed on eviction."""
+        try:
+            self._owned.add(client)
+        except TypeError:
+            pass  # values that cannot be weak-referenced are never litellm clients
+
+    def _is_owned(self, client: object) -> bool:
+        try:
+            return client in self._owned
+        except TypeError:
+            return False  # unhashable values are never litellm clients
+
+    def schedule(self, client: object) -> None:
+        """Queue an evicted client for closing once it is idle and out of grace.
+
+        Past ``max_pending`` the client is left to the collector instead, so a
+        workload that churns the cache cannot grow this queue without bound.
+        Every queued entry comes due within one grace window, so the capacity it
+        occupies is returned within that window rather than held.
+        """
+        if client is None or not self._is_owned(client):
+            return
+        close_fn: Final = _close_function(client)
+        if close_fn is None:
+            return
+        if self._pending_count >= self._max_pending:
+            return
+        self._enqueue(
+            _PendingClose(
+                client_ref=weakref.ref(client),
+                loop_id=_running_loop_id(),
+                needs_loop=inspect.iscoroutinefunction(close_fn),
+                close_after=self._clock() + self._grace_seconds,
+            )
+        )
+
+    def reap(self) -> None:
+        """Close every queued client that is due, idle, and closable from here.
+
+        Called from the cache's read path, so the empty-queue exit comes first and
+        the work done past it is proportional to what is due, not to the queue.
+        """
+        if not self._pending_count:
+            return
+        now: Final = self._clock()
+        for pending in self._take_due(_running_loop_id(), now):
+            client = pending.client_ref()
+            if client is None:
+                continue
+            if _has_connection_in_flight(client):
+                self._enqueue(replace(pending, close_after=now + self._grace_seconds))
+                continue
+            self._close(client)
+
+    @property
+    def pending_count(self) -> int:
+        return self._pending_count
+
+    def _enqueue(self, pending: _PendingClose) -> None:
+        """Append to the entry's bucket, dropping any dead entries it queues behind.
+
+        Deadlines only ever move forward, so appending keeps each bucket ordered
+        by deadline, and entries whose client the collector already took sit at
+        the front rather than having to be searched for.
+        """
+        with self._queue_lock:
+            bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())  # mutable-ok: FIFO by design
+            while bucket and bucket[0].client_ref() is None:
+                bucket.popleft()
+                self._pending_count -= 1
+            bucket.append(pending)
+            self._pending_count += 1
+
+    def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
+        buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
+        with self._queue_lock:
+            return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
+
+    def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
+        bucket: Final = self._buckets.get(key)
+        if bucket is None:
+            return
+        while bucket and bucket[0].close_after <= now:
+            self._pending_count -= 1
+            yield bucket.popleft()
+        if not bucket:
+            del self._buckets[key]
+
+    def _close(self, client: object) -> None:
+        close_fn: Final = _close_function(client)
+        if close_fn is None:
+            return
+        try:
+            closing: Final = close_fn()
+        except Exception:  # noqa: BLE001 - a discarded client's close must never surface to callers
+            return
+        if not inspect.isawaitable(closing):
+            return
+        task: Final = asyncio.get_running_loop().create_task(_close_quietly(closing))
+        self._close_tasks.add(task)
+        task.add_done_callback(self._close_tasks.discard)
+
+
+default_evicted_client_closer: Final = EvictedClientCloser()

litellm/caching/llm_caching_handler.py

@@ -5,21 +5,44 @@ Add the event loop to the cache key, to prevent event loop closed errors.
 import asyncio
 from typing import Final
 
+from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
 from .in_memory_cache import InMemoryCache
 
 
 class LLMClientCache(InMemoryCache):
     """Cache for LLM HTTP clients (OpenAI, Azure, httpx, etc.).
 
-    IMPORTANT: This cache intentionally does NOT close clients on eviction.
-    Evicted clients may still be in use by in-flight requests. Closing them
-    eagerly causes ``RuntimeError: Cannot send a request, as the client has
-    been closed.`` errors in production after the TTL (1 hour) expires.
+    An evicted client is never closed on the spot: a request handed the client
+    just before eviction is still using it, and closing it there raises
+    ``RuntimeError: Cannot send a request, as the client has been closed.``
 
-    Clients that are no longer referenced will be garbage-collected normally.
-    For explicit shutdown cleanup, use ``close_litellm_async_clients()``.
+    Nor can eviction be left to rely on garbage collection. The SDK clients are
+    reference cycles, so an evicted client and its open TCP connections survive
+    until a generational collection runs. Instead a client litellm created is
+    handed to ``EvictedClientCloser``, which closes it once a grace window has
+    passed. Clients the caller supplied are left untouched.
     """
 
+    def __init__(
+        self,
+        max_size_in_memory: int | None = 200,
+        default_ttl: int | None = 600,
+        max_size_per_item: int | None = 1024,
+        evicted_client_closer: EvictedClientCloser | None = None,
+    ) -> None:
+        super().__init__(
+            max_size_in_memory=max_size_in_memory,
+            default_ttl=default_ttl,
+            max_size_per_item=max_size_per_item,
+        )
+        self.evicted_client_closer = evicted_client_closer or default_evicted_client_closer
+
+    def _remove_key(self, key: str) -> None:
+        evicted: Final[object] = self.cache_dict.get(key)
+        super()._remove_key(key)
+        self.evicted_client_closer.schedule(evicted)
+        self.evicted_client_closer.reap()
+
     def update_cache_key_with_event_loop(self, key):
         """
         Add the event loop to the cache key, to prevent event loop closed errors.
@@ -32,16 +55,22 @@ class LLMClientCache(InMemoryCache):
         except RuntimeError:  # handle no current running event loop
             return key
 
-    def set_cache(self, key, value, **kwargs):
+    def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
+        """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
+        if litellm_owned_client:
+            self.evicted_client_closer.mark_owned(value)
         key = self.update_cache_key_with_event_loop(key)
         return super().set_cache(key, value, **kwargs)
 
-    async def async_set_cache(self, key, value, **kwargs):
+    async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
+        if litellm_owned_client:
+            self.evicted_client_closer.mark_owned(value)
         key = self.update_cache_key_with_event_loop(key)
         return await super().async_set_cache(key, value, **kwargs)
 
     def get_cache(self, key, **kwargs):
         key = self.update_cache_key_with_event_loop(key)
+        self.evicted_client_closer.reap()
 
         return super().get_cache(key, **kwargs)
 

litellm/caching/redis_cache.py

@@ -18,7 +18,7 @@ import time
 from collections.abc import Awaitable, Callable, Sequence
 from contextvars import ContextVar
 from datetime import timedelta
-from typing import TYPE_CHECKING, Any, Final, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Final, TypeVar, cast
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -49,7 +49,7 @@ if TYPE_CHECKING:
     cluster_pipeline = ClusterPipeline
     async_redis_client = Redis
     async_redis_cluster_client = RedisCluster
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     pipeline = Any
     cluster_pipeline = Any

litellm/caching/redis_cluster_cache.py

@@ -5,7 +5,7 @@ Key differences:
 - RedisClient NEEDs to be re-used across requests, adds 3000ms latency if it's re-created
 """
 
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.caching.redis_cache import RedisCache
 
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 
     pipeline = Pipeline
     async_redis_client = Redis
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     pipeline = Any
     async_redis_client = Any

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -367,7 +367,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 stream_options = normalize_responses_api_stream_options(value)
                 if stream_options is not None:
                     responses_api_request["stream_options"] = stream_options
-            elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
+            elif key in ResponsesAPIOptionalRequestParams.__annotations__:
                 responses_api_request[key] = value
             elif key == "previous_response_id":
                 responses_api_request["previous_response_id"] = value

litellm/constants.py

@@ -197,6 +197,16 @@ RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600))  # 10
 ########## Networking constants ##############################################################
 _DEFAULT_TTL_FOR_HTTPX_CLIENTS: Final = 3600  # 1 hour, re-use the same httpx client for 1 hour
 
+# The earliest an evicted, litellm-created client may be closed. A request handed the
+# client just before eviction is still using it, so nothing is closed inside this window;
+# past it, the client is closed once it reports no connection in flight.
+EVICTED_LLM_CLIENT_CLOSE_GRACE_SECONDS: Final = 900
+
+# How many evicted clients may be queued for closing at once. Past this, an evicted client
+# is left to the collector rather than letting a cache-churning workload grow the queue
+# without bound. Each queued entry is ~100 bytes and comes due within one grace window.
+EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000
+
 # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
 # Set to 0 for unlimited (not recommended for production)
 AIOHTTP_CONNECTOR_LIMIT: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000))

litellm/containers/__init__.py

@@ -23,22 +23,20 @@ from .main import (
 )
 
 __all__ = [
-    # Core container operations
     "acreate_container",
     "adelete_container",
-    "alist_containers",
-    "aretrieve_container",
-    "create_container",
-    "delete_container",
-    "list_containers",
-    "retrieve_container",
-    # Container file operations (auto-generated from endpoints.json)
     "adelete_container_file",
     "alist_container_files",
+    "alist_containers",
+    "aretrieve_container",
     "aretrieve_container_file",
     "aretrieve_container_file_content",
+    "create_container",
+    "delete_container",
     "delete_container_file",
     "list_container_files",
+    "list_containers",
+    "retrieve_container",
     "retrieve_container_file",
     "retrieve_container_file_content",
 ]

litellm/fine_tuning/main.py

@@ -80,7 +80,7 @@ async def acreate_fine_tuning_job(
     hyperparameters: dict | None = {},
     suffix: str | None = None,
     validation_file: str | None = None,
-    integrations: List[str] | None = None,
+    integrations: list[str] | None = None,
     seed: int | None = None,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
     extra_headers: dict[str, str] | None = None,
@@ -157,7 +157,7 @@ def create_fine_tuning_job(
     hyperparameters: dict | None = {},
     suffix: str | None = None,
     validation_file: str | None = None,
-    integrations: List[str] | None = None,
+    integrations: list[str] | None = None,
     seed: int | None = None,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
     extra_headers: dict[str, str] | None = None,

litellm/integrations/arize/arize.py

@@ -6,7 +6,7 @@ this file has Arize ai specific helper functions
 
 import os
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.integrations.arize import _utils
 from litellm.integrations.arize._utils import ArizeOTELAttributes
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
     from litellm.types.integrations.arize import Protocol as _Protocol
 
     Protocol = _Protocol
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Protocol = Any
     Span = Any

litellm/integrations/arize/arize_phoenix.py

@@ -1,7 +1,7 @@
 import os
 import threading
 from collections import OrderedDict
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
 from litellm.integrations.arize import _utils
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
 
     Protocol = _Protocol
     OpenTelemetryConfig = _OpenTelemetryConfig
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     OpenTelemetry = _OpenTelemetry
     LITELLM_TRACER_NAME: str
 else:

litellm/integrations/custom_logger.py

@@ -3,7 +3,7 @@
 import re
 import traceback
 from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, Final, Optional, Union
+from typing import TYPE_CHECKING, Any, Final, Optional
 
 from pydantic import BaseModel
 
@@ -39,7 +39,7 @@ if TYPE_CHECKING:
     )
     from litellm.types.router import PreRoutingHookResponse
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
     LiteLLMLoggingObj = Any

litellm/integrations/dotprompt/prompt_manager.py

@@ -31,7 +31,7 @@ class PromptTemplate:
         self.output_format = self.metadata.get("output", {}).get("format")
         self.output_schema = self.metadata.get("output", {}).get("schema", {})
         self.optional_params = {}
-        for key in self.metadata.keys():
+        for key in self.metadata:
             if key not in restricted_keys:
                 self.optional_params[key] = self.metadata[key]
 

litellm/integrations/langfuse/langfuse_otel.py

@@ -2,7 +2,7 @@ import base64
 import json
 import os
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Optional, Union
+from typing import TYPE_CHECKING, Any, Final, Optional
 
 from litellm._logging import verbose_logger
 from litellm.integrations.arize import _utils
@@ -18,7 +18,7 @@ from litellm.types.utils import StandardCallbackDynamicParams
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -4,7 +4,7 @@ Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
 
 import os
 from functools import lru_cache
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
 from packaging.version import Version
 
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
 
     LangfuseClass: TypeAlias = Langfuse
 
-    PROMPT_CLIENT = Union[TextPromptClient, ChatPromptClient]
+    PROMPT_CLIENT = TextPromptClient | ChatPromptClient
 else:
     PROMPT_CLIENT = Any
     LangfuseClass = Any

litellm/integrations/langtrace.py

@@ -1,12 +1,12 @@
 import json
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.proxy._types import SpanAttributes
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/integrations/levo/levo.py

@@ -1,5 +1,5 @@
 import os
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.integrations.opentelemetry import OpenTelemetry
 
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
 
     Protocol = _Protocol
     OpenTelemetryConfig = _OpenTelemetryConfig
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Protocol = Any
     OpenTelemetryConfig = Any

litellm/integrations/opentelemetry.py

@@ -1,7 +1,7 @@
 import os
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Union, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import verbose_logger
@@ -47,12 +47,12 @@ if TYPE_CHECKING:
     )
     from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
 
-    Span = Union[_Span, Any]
-    Tracer = Union[_Tracer, Any]
-    Context = Union[_Context, Any]
-    SpanExporter = Union[_SpanExporter, Any]
-    UserAPIKeyAuth = Union[_UserAPIKeyAuth, Any]
-    ManagementEndpointLoggingPayload = Union[_ManagementEndpointLoggingPayload, Any]
+    Span = _Span | Any
+    Tracer = _Tracer | Any
+    Context = _Context | Any
+    SpanExporter = _SpanExporter | Any
+    UserAPIKeyAuth = _UserAPIKeyAuth | Any
+    ManagementEndpointLoggingPayload = _ManagementEndpointLoggingPayload | Any
 else:
     Span = Any
     Tracer = Any
@@ -186,16 +186,7 @@ def _normalize_team_metadata_keys(value: Any) -> list[str]:
 
 _FREEZE_MAX_DEPTH: Final = 16
 
-HashableScope = Union[
-    str,
-    int,
-    float,
-    bool,
-    bytes,
-    None,
-    tuple["HashableScope", ...],
-    frozenset["HashableScope"],
-]
+HashableScope = str | int | float | bool | bytes | None | tuple["HashableScope", ...] | frozenset["HashableScope"]
 
 
 def _freeze_for_dedupe(value: object, _depth: int = 0) -> HashableScope:

litellm/integrations/opentelemetry_utils/gen_ai_semconv.py

@@ -31,7 +31,7 @@ Events:
 
 from datetime import datetime
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
@@ -40,7 +40,7 @@ if TYPE_CHECKING:
 
     from litellm.integrations.opentelemetry import OpenTelemetryConfig
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/integrations/opik/opik_payload_builder/types.py

@@ -1,7 +1,7 @@
 """Type definitions for Opik payload building."""
 
 from dataclasses import dataclass
-from typing import Any, Final, Literal, Union
+from typing import Any, Final, Literal
 
 
 @dataclass
@@ -42,5 +42,5 @@ class SpanPayload:
     total_cost: float | None = None
 
 
-PayloadItem = Union[TracePayload, SpanPayload]
+PayloadItem = TracePayload | SpanPayload
 TraceSpanPayloadTuple: Final = tuple[TracePayload | None, SpanPayload]

litellm/integrations/otel/__init__.py

@@ -72,53 +72,49 @@ from litellm.integrations.otel.model.spans import (
 )
 
 __all__ = [
-    # config
-    "OTEL_V2_ENV",
-    "OpenTelemetryV2Config",
-    "is_otel_v2_enabled",
-    # semconv
     "BAGGAGE_PROMOTED_KEYS",
     "DB",
     "DEFAULT_BAGGAGE_METADATA_KEYS",
+    "HTTP",
+    "MCP",
+    "OTEL_V2_ENV",
+    "SPAN_REGISTRY",
     "Client",
     "Error",
     "GenAI",
     "GenAIOperation",
     "GenAIProvider",
-    "HTTP",
-    "JsonRpc",
-    "LiteLLM",
-    "LiteLLMError",
-    "MCP",
-    "MCPMethod",
-    "Metric",
-    "Network",
-    "NetworkTransport",
-    "Server",
-    "resolve_operation",
-    "resolve_provider",
-    # spans
-    "SPAN_REGISTRY",
-    "LiteLLMSpanKind",
-    "SpanRole",
-    "SpanSpec",
-    "db_system",
-    "span_role_for_service",
-    "validate_registry",
-    # payloads
     "GuardrailSpanData",
+    "JsonRpc",
     "LLMCallSpanData",
     "LLMRequestParams",
     "LLMUsage",
+    "LiteLLM",
+    "LiteLLMError",
+    "LiteLLMSpanKind",
     "MCPListToolsSpanData",
+    "MCPMethod",
     "MCPToolCallSpanData",
+    "Metric",
+    "Network",
+    "NetworkTransport",
+    "OpenTelemetryV2Config",
     "ProxyRequestSpanData",
     "RequestContext",
     "RequestIdentity",
+    "Server",
     "ServerInfo",
     "ServiceSpanData",
     "SpanError",
+    "SpanRole",
+    "SpanSpec",
+    "db_system",
     "is_mcp_list_tools",
     "is_mcp_tool_call",
+    "is_otel_v2_enabled",
     "promoted_baggage",
+    "resolve_operation",
+    "resolve_provider",
+    "span_role_for_service",
+    "validate_registry",
 ]

litellm/interactions/__init__.py

@@ -66,18 +66,13 @@ from litellm.interactions.main import (
 )
 
 __all__ = [
-    # Create
-    "create",
-    "acreate",
-    # Get
-    "get",
-    "aget",
-    # Delete
-    "delete",
-    "adelete",
-    # Cancel
-    "cancel",
     "acancel",
-    # Sub-modules
+    "acreate",
+    "adelete",
     "agents",
+    "aget",
+    "cancel",
+    "create",
+    "delete",
+    "get",
 ]

litellm/litellm_core_utils/core_helpers.py

@@ -2,7 +2,7 @@
 ## Helper utilities
 import copy
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Final, Literal, Union
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
 
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
 
     from litellm.types.utils import ModelResponseStream
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/litellm_core_utils/fallback_generalizations.py

@@ -48,7 +48,7 @@ O(number of rules); callers must only invoke them on a cache miss.
 
 import re
 from dataclasses import dataclass
-from typing import Final, Union
+from typing import Final
 
 from litellm._logging import verbose_logger
 
@@ -100,7 +100,7 @@ class _CapabilityRule:
     model_info: dict
 
 
-_CompiledRule = Union[_RoutingRule, _CapabilityRule]
+_CompiledRule = _RoutingRule | _CapabilityRule
 
 
 def _compile_rule(rule: object) -> tuple[_CompiledRule, ...]:

litellm/litellm_core_utils/litellm_logging.py

@@ -4827,7 +4827,7 @@ class StandardLoggingPayloadSetup:
 
         # Populate well-known typed fields with int/str coercion where needed
         typed_keys: Final[dict] = {}
-        for key in StandardLoggingAdditionalHeaders.__annotations__.keys():
+        for key in StandardLoggingAdditionalHeaders.__annotations__:
             _key = key.lower().replace("_", "-")
             typed_keys[_key] = key
             if _key in additiona_headers:
@@ -4859,7 +4859,7 @@ class StandardLoggingPayloadSetup:
             usage_object=None,
         )
         if hidden_params is not None:
-            for key in StandardLoggingHiddenParams.__annotations__.keys():
+            for key in StandardLoggingHiddenParams.__annotations__:
                 if key in hidden_params:
                     if key == "additional_headers":
                         clean_hidden_params["additional_headers"] = StandardLoggingPayloadSetup.get_additional_headers(
@@ -5501,7 +5501,7 @@ def get_standard_logging_metadata(
     )
     if isinstance(metadata, dict):
         # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields
-        for key in StandardLoggingMetadata.__annotations__.keys():
+        for key in StandardLoggingMetadata.__annotations__:
             if key in metadata:
                 clean_metadata[key] = metadata[key]
 

litellm/litellm_core_utils/logging_utils.py

@@ -4,7 +4,7 @@ import inspect
 import re
 import time
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
 from litellm.constants import MAX_BASE64_LENGTH_FOR_LOGGING
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
     )
 
     LiteLLMModelResponse = _ModelResponse
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     LiteLLMModelResponse = Any
     LiteLLMLoggingObject = Any

litellm/litellm_core_utils/model_response_utils.py

@@ -47,7 +47,7 @@ def is_model_response_stream_empty(model_response: ModelResponseStream) -> bool:
 
     # Check for any non-base fields that are set
     # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
-    for model_response_field in type(model_response).model_fields.keys():
+    for model_response_field in type(model_response).model_fields:
         # Skip base fields that are always set
         if model_response_field in BASE_FIELDS:
             continue

litellm/litellm_core_utils/streaming_handler.py

@@ -8,7 +8,7 @@ import time
 import traceback
 from collections.abc import AsyncIterator, Callable, Iterator
 from dataclasses import dataclass
-from typing import Any, Final, NoReturn, TypeVar, Union, cast
+from typing import Any, Final, NoReturn, TypeVar, cast
 
 import anyio
 import httpx
@@ -99,7 +99,7 @@ class _ProviderChunkEarlyReturn:
     value: Any
 
 
-_ProviderChunkResult = Union[_ProviderChunkParsed, _ProviderChunkEarlyReturn]
+_ProviderChunkResult = _ProviderChunkParsed | _ProviderChunkEarlyReturn
 
 
 class CustomStreamWrapper:
@@ -256,9 +256,7 @@ class CustomStreamWrapper:
             chunk = chunk.strip()
             self.complete_response = self.complete_response.strip()
 
-            if chunk.startswith(self.complete_response):
-                # Remove last_sent_chunk only if it appears at the start of the new chunk
-                chunk = chunk[len(self.complete_response) :]
+            chunk = chunk.removeprefix(self.complete_response)
 
             self.complete_response += chunk
             return chunk

litellm/llms/azure/common_utils.py

@@ -427,88 +427,95 @@ class BaseAzureLLM(BaseOpenAILLM):
             f"|azure_password={hashlib.sha256(_azure_password.encode()).hexdigest() if isinstance(_azure_password, str) else None}"
             f"|azure_scope={_lp.get('azure_scope')}"
         )
-        if client is None:
-            cached_client: Final = self.get_cached_openai_client(
-                client_initialization_params=client_initialization_params,
-                client_type="azure",
-            )
-            if cached_client:
-                if isinstance(cached_client, (AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI)):
-                    return cached_client
-
-            azure_client_params: Final = self.initialize_azure_sdk_client(
-                litellm_params=litellm_params or {},
-                api_key=api_key,
-                api_base=api_base,
-                model_name=model,
-                api_version=api_version,
-                is_async=_is_async,
-            )
-
-            # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
-            # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
-            if self._is_azure_v1_api_version(api_version):
-                # Extract only params that OpenAI client accepts
-                # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
-                # The OpenAI client accepts a callable for `api_key` and re-invokes it
-                # on every request (via `_refresh_api_key`), so passing
-                # `azure_ad_token_provider` directly preserves Azure AD token refresh
-                # behavior that the regular AzureOpenAI client provides.
-                v1_api_key: str | Callable[[], Any] | None = (
-                    azure_client_params.get("api_key")
-                    or azure_client_params.get("azure_ad_token_provider")
-                    or azure_client_params.get("azure_ad_token")
-                )
-                if _is_async is True and callable(v1_api_key):
-                    # AsyncOpenAI expects an async provider; wrap the sync provider
-                    # returned by azure-identity. Offload to a thread so a token
-                    # refresh (blocking HTTP call to AAD on cache miss) does not
-                    # stall the event loop.
-                    _sync_provider: Final = v1_api_key
-
-                    async def _async_v1_api_key() -> str:
-                        return await asyncio.to_thread(_sync_provider)
-
-                    v1_api_key = _async_v1_api_key
-
-                v1_params: Final[dict[str, Any]] = {
-                    "api_key": v1_api_key,
-                    "base_url": f"{api_base}/openai/v1/",
-                }
-                if "timeout" in azure_client_params:
-                    v1_params["timeout"] = azure_client_params["timeout"]
-                if "max_retries" in azure_client_params:
-                    v1_params["max_retries"] = azure_client_params["max_retries"]
-                if "http_client" in azure_client_params:
-                    v1_params["http_client"] = azure_client_params["http_client"]
-
-                verbose_logger.debug("Using Azure v1 API with base_url: %s", v1_params["base_url"])
-
-                if _is_async is True:
-                    openai_client = AsyncOpenAI(**v1_params)
-                else:
-                    openai_client = OpenAI(**v1_params)
-            else:
-                # Traditional Azure API uses AzureOpenAI client
-                if _is_async is True:
-                    openai_client = AsyncAzureOpenAI(**azure_client_params)
-                else:
-                    openai_client = AzureOpenAI(**azure_client_params)
-        else:
-            openai_client = client
+        if client is not None:
             if (
                 api_version is not None
-                and isinstance(openai_client, (AzureOpenAI, AsyncAzureOpenAI))
-                and isinstance(openai_client._custom_query, dict)
+                and isinstance(client, (AzureOpenAI, AsyncAzureOpenAI))
+                and isinstance(client._custom_query, dict)
             ):
                 # set api_version to version passed by user
-                openai_client._custom_query.setdefault("api-version", api_version)
+                client._custom_query.setdefault("api-version", api_version)
+            self.set_cached_openai_client(
+                openai_client=client,
+                client_initialization_params=client_initialization_params,
+                client_type="azure",
+                litellm_owned_client=False,
+            )
+            return client
+
+        cached_client: Final = self.get_cached_openai_client(
+            client_initialization_params=client_initialization_params,
+            client_type="azure",
+        )
+        if cached_client:
+            if isinstance(cached_client, (AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI)):
+                return cached_client
+
+        azure_client_params: Final = self.initialize_azure_sdk_client(
+            litellm_params=litellm_params or {},
+            api_key=api_key,
+            api_base=api_base,
+            model_name=model,
+            api_version=api_version,
+            is_async=_is_async,
+        )
+
+        # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
+        # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
+        if self._is_azure_v1_api_version(api_version):
+            # Extract only params that OpenAI client accepts
+            # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
+            # The OpenAI client accepts a callable for `api_key` and re-invokes it
+            # on every request (via `_refresh_api_key`), so passing
+            # `azure_ad_token_provider` directly preserves Azure AD token refresh
+            # behavior that the regular AzureOpenAI client provides.
+            v1_api_key: str | Callable[[], Any] | None = (
+                azure_client_params.get("api_key")
+                or azure_client_params.get("azure_ad_token_provider")
+                or azure_client_params.get("azure_ad_token")
+            )
+            if _is_async is True and callable(v1_api_key):
+                # AsyncOpenAI expects an async provider; wrap the sync provider
+                # returned by azure-identity. Offload to a thread so a token
+                # refresh (blocking HTTP call to AAD on cache miss) does not
+                # stall the event loop.
+                _sync_provider: Final = v1_api_key
+
+                async def _async_v1_api_key() -> str:
+                    return await asyncio.to_thread(_sync_provider)
+
+                v1_api_key = _async_v1_api_key
+
+            v1_params: Final[dict[str, Any]] = {
+                "api_key": v1_api_key,
+                "base_url": f"{api_base}/openai/v1/",
+            }
+            if "timeout" in azure_client_params:
+                v1_params["timeout"] = azure_client_params["timeout"]
+            if "max_retries" in azure_client_params:
+                v1_params["max_retries"] = azure_client_params["max_retries"]
+            if "http_client" in azure_client_params:
+                v1_params["http_client"] = azure_client_params["http_client"]
+
+            verbose_logger.debug("Using Azure v1 API with base_url: %s", v1_params["base_url"])
+
+            if _is_async is True:
+                openai_client = AsyncOpenAI(**v1_params)
+            else:
+                openai_client = OpenAI(**v1_params)
+        else:
+            # Traditional Azure API uses AzureOpenAI client
+            if _is_async is True:
+                openai_client = AsyncAzureOpenAI(**azure_client_params)
+            else:
+                openai_client = AzureOpenAI(**azure_client_params)
 
         # save client in-memory cache
         self.set_cached_openai_client(
             openai_client=openai_client,
             client_initialization_params=client_initialization_params,
             client_type="azure",
+            litellm_owned_client=self.owns_wrapped_http_client(azure_client_params.get("http_client")),
         )
         return openai_client
 

litellm/llms/base_llm/managed_resources/base_managed_resource.py

@@ -5,7 +5,7 @@
 import base64
 import json
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Generic, TypeVar, cast
 
 from litellm import verbose_logger
 from litellm.llms.base_llm.managed_resources.isolation import (
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient as _PrismaClient
     from litellm.router import Router as _Router
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     InternalUsageCache = _InternalUsageCache
     PrismaClient = _PrismaClient
     Router = _Router

litellm/llms/bedrock/chat/invoke_handler.py

@@ -57,7 +57,7 @@ class AmazonCohereChatConfig:
     Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html
     """
 
-    documents: List[Document] | None = None
+    documents: list[Document] | None = None
     search_queries_only: bool | None = None
     preamble: str | None = None
     max_tokens: int | None = None
@@ -69,12 +69,12 @@ class AmazonCohereChatConfig:
     presence_penalty: float | None = None
     seed: int | None = None
     return_prompt: bool | None = None
-    stop_sequences: List[str] | None = None
+    stop_sequences: list[str] | None = None
     raw_prompting: bool | None = None
 
     def __init__(
         self,
-        documents: List[Document] | None = None,
+        documents: list[Document] | None = None,
         search_queries_only: bool | None = None,
         preamble: str | None = None,
         max_tokens: int | None = None,
@@ -112,7 +112,7 @@ class AmazonCohereChatConfig:
             and v is not None
         }
 
-    def get_supported_openai_params(self) -> List[str]:
+    def get_supported_openai_params(self) -> list[str]:
         return [
             "max_tokens",
             "max_completion_tokens",
@@ -325,7 +325,7 @@ class AWSEventStreamDecoder:
 
         self.model = model
         self.parser = EventStreamJSONParser()
-        self.content_blocks: List[ContentBlockDeltaEvent] = []
+        self.content_blocks: list[ContentBlockDeltaEvent] = []
         self.tool_calls_index: int | None = None
         self.response_id: str | None = None
         self.json_mode = json_mode
@@ -362,13 +362,13 @@ class AWSEventStreamDecoder:
 
     def translate_thinking_blocks(
         self, thinking_block: BedrockConverseReasoningContentBlockDelta
-    ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None:
+    ) -> list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None:
         """
         Translate the thinking blocks to a string
         """
 
-        thinking_blocks_list: Final[List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]] = []
-        _thinking_block: Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] | None = None
+        thinking_blocks_list: Final[list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock]] = []
+        _thinking_block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None = None
 
         if "text" in thinking_block:
             _thinking_block = ChatCompletionThinkingBlock(type="thinking")
@@ -402,12 +402,12 @@ class AWSEventStreamDecoder:
     ) -> tuple[
         ChatCompletionToolCallChunk | None,
         dict,
-        List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None,
+        list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None,
     ]:
         """Handle 'start' event in converse chunk parsing."""
         tool_use: ChatCompletionToolCallChunk | None = None
         provider_specific_fields: dict = {}
-        thinking_blocks: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None = None
+        thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
 
         self.content_blocks = []  # reset
         if start_obj is not None:
@@ -450,14 +450,14 @@ class AWSEventStreamDecoder:
         ChatCompletionToolCallChunk | None,
         dict,
         str | None,
-        List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None,
+        list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None,
     ]:
         """Handle 'delta' event in converse chunk parsing."""
         text = ""
         tool_use: ChatCompletionToolCallChunk | None = None
         provider_specific_fields: dict = {}
         reasoning_content: str | None = None
-        thinking_blocks: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None = None
+        thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
 
         self.content_blocks.append(delta_obj)
         if "text" in delta_obj:
@@ -535,7 +535,7 @@ class AWSEventStreamDecoder:
             usage: Usage | None = None
             provider_specific_fields: dict = {}
             reasoning_content: str | None = None
-            thinking_blocks: List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] | None = None
+            thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None
 
             content_block_index: Final = int(chunk_data.get("contentBlockIndex", 0))
             if "start" in chunk_data:
@@ -590,7 +590,7 @@ class AWSEventStreamDecoder:
         except Exception as e:
             raise Exception(f"Received streaming error - {e}")
 
-    def _chunk_parser(self, chunk_data: dict) -> Union[GChunk, ModelResponseStream, dict]:
+    def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict:
         text = ""
         is_finished = False
         finish_reason = ""
@@ -645,7 +645,7 @@ class AWSEventStreamDecoder:
             tool_use=None,
         )
 
-    def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[Union[GChunk, ModelResponseStream, dict]]:
+    def iter_bytes(self, iterator: Iterator[bytes]) -> Iterator[GChunk | ModelResponseStream | dict]:
         """Given an iterator that yields lines, iterate over it & yield every event encountered"""
         from botocore.eventstream import EventStreamBuffer
 
@@ -659,9 +659,7 @@ class AWSEventStreamDecoder:
                     _data = json.loads(message)
                     yield self._chunk_parser(chunk_data=_data)
 
-    async def aiter_bytes(
-        self, iterator: AsyncIterator[bytes]
-    ) -> AsyncIterator[Union[GChunk, ModelResponseStream, dict]]:
+    async def aiter_bytes(self, iterator: AsyncIterator[bytes]) -> AsyncIterator[GChunk | ModelResponseStream | dict]:
         """Given an async iterator that yields lines, iterate over it & yield every event encountered"""
         from botocore.eventstream import EventStreamBuffer
 
@@ -741,7 +739,7 @@ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
             sync_stream=sync_stream,
         )
 
-    def _chunk_parser(self, chunk_data: dict) -> Union[GChunk, ModelResponseStream, dict]:
+    def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict:
         return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
 
 
@@ -756,7 +754,7 @@ class MockResponseIterator:  # for returning ai21 streaming responses
         return self
 
     def _handle_json_mode_chunk(
-        self, text: str, tool_calls: List[ChatCompletionToolCallChunk] | None
+        self, text: str, tool_calls: list[ChatCompletionToolCallChunk] | None
     ) -> tuple[str, ChatCompletionToolCallChunk | None]:
         """
         If JSON mode is enabled, convert the tool call to a message.
@@ -789,7 +787,7 @@ class MockResponseIterator:  # for returning ai21 streaming responses
             text = chunk_data.choices[0].message.content or ""
             tool_use = None
             _model_response_tool_call: Final = cast(
-                List[ChatCompletionMessageToolCall] | None,
+                list[ChatCompletionMessageToolCall] | None,
                 cast(Choices, chunk_data.choices[0]).message.tool_calls,
             )
             if self.json_mode is True:

litellm/llms/bedrock/embed/cohere_transformation.py

@@ -34,7 +34,7 @@ class BedrockCohereEmbeddingConfig:
         new_transformed_request: Final = CohereEmbeddingRequest(
             input_type=transformed_request["input_type"],
         )
-        for k in CohereEmbeddingRequest.__annotations__.keys():
+        for k in CohereEmbeddingRequest.__annotations__:
             if k in transformed_request:
                 new_transformed_request[k] = transformed_request[k]
 

litellm/llms/bedrock/image_generation/image_handler.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import json
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 import httpx
 from pydantic import BaseModel
@@ -49,12 +49,12 @@ class BedrockImagePreparedRequest(BaseModel):
     data: dict
 
 
-BedrockImageConfigClass = Union[
-    type[AmazonTitanImageGenerationConfig],
-    type[AmazonNovaCanvasConfig],
-    type[AmazonStability3Config],
-    type[AmazonStabilityConfig],
-]
+BedrockImageConfigClass = (
+    type[AmazonTitanImageGenerationConfig]
+    | type[AmazonNovaCanvasConfig]
+    | type[AmazonStability3Config]
+    | type[AmazonStabilityConfig]
+)
 
 
 class BedrockImageGeneration(BaseAWSLLM):

litellm/llms/bedrock/vector_stores/transformation.py

@@ -160,10 +160,10 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
                 aws_filters: dict | None = None
 
                 if isinstance(value, dict):
-                    if "operator" in value.keys():
+                    if "operator" in value:
                         # Single operator - map directly (no wrapping needed)
                         aws_filters = self._map_operator_filter(value)
-                    elif "and" in value.keys() or "or" in value.keys():
+                    elif "and" in value or "or" in value:
                         aws_filters = self._map_and_or_filters(value)
                     else:
                         # Assume it's already in AWS KB format

litellm/llms/custom_httpx/http_handler.py

@@ -1441,6 +1441,7 @@ def get_async_httpx_client(
         key=_cache_key_name,
         value=_new_client,
         ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+        litellm_owned_client=True,
     )
     return _new_client
 
@@ -1486,5 +1487,6 @@ def _get_httpx_client(params: dict | None = None) -> HTTPHandler:
         key=_cache_key_name,
         value=_new_client,
         ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+        litellm_owned_client=True,
     )
     return _new_client

litellm/llms/openai/common_utils.py

@@ -128,13 +128,33 @@ class BaseOpenAILLM:
         _cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
         return _cached_client
 
+    @staticmethod
+    def owns_wrapped_http_client(http_client: httpx.Client | httpx.AsyncClient | None) -> bool:
+        """Whether litellm may close an SDK client built around ``http_client``.
+
+        ``_get_async_http_client`` / ``_get_sync_http_client`` hand back
+        ``litellm.aclient_session`` / ``litellm.client_session`` when the caller
+        configured one. The SDK's ``close()`` closes whatever http client it was
+        given, so an SDK client wrapping one of those shared sessions must never be
+        closed on eviction; the caller goes on using the session. ``None`` means the
+        SDK built its own http client, which litellm does own.
+        """
+        if http_client is None:
+            return True
+        return http_client is not litellm.aclient_session and http_client is not litellm.client_session
+
     @staticmethod
     def set_cached_openai_client(
         openai_client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI,
         client_type: Literal["openai", "azure"],
         client_initialization_params: dict,
+        litellm_owned_client: bool = False,
     ):
-        """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS"""
+        """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS
+
+        ``litellm_owned_client`` says litellm built this client, so the cache may close it once it
+        is evicted. A client the caller supplied stays open, since litellm does not own it.
+        """
         _cache_key: Final = BaseOpenAILLM.get_openai_client_cache_key(
             client_initialization_params=client_initialization_params,
             client_type=client_type,
@@ -143,6 +163,7 @@ class BaseOpenAILLM:
             key=_cache_key,
             value=openai_client,
             ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+            litellm_owned_client=litellm_owned_client,
         )
 
     @staticmethod

litellm/llms/openai/openai.py

@@ -345,7 +345,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         client: OpenAI | AsyncOpenAI | None = None,
         shared_session: Optional["ClientSession"] = None,
     ) -> OpenAI | AsyncOpenAI | None:
-        client_initialization_params: Final[Dict] = locals()
+        client_initialization_params: Final[dict] = locals()
         if client is None:
             if not isinstance(max_retries, int):
                 raise OpenAIError(
@@ -360,11 +360,16 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             if cached_client:
                 if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI):
                     return cached_client
+            http_client: Final[httpx.Client | httpx.AsyncClient | None] = (
+                OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
+                if is_async
+                else OpenAIChatCompletion._get_sync_http_client()
+            )
             if is_async:
                 _new_client: OpenAI | AsyncOpenAI = AsyncOpenAI(
                     api_key=api_key,
                     base_url=api_base,
-                    http_client=OpenAIChatCompletion._get_async_http_client(shared_session=shared_session),
+                    http_client=http_client,
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
@@ -373,7 +378,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 _new_client = OpenAI(
                     api_key=api_key,
                     base_url=api_base,
-                    http_client=OpenAIChatCompletion._get_sync_http_client(),
+                    http_client=http_client,
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
@@ -384,6 +389,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 openai_client=_new_client,
                 client_initialization_params=client_initialization_params,
                 client_type="openai",
+                litellm_owned_client=self.owns_wrapped_http_client(http_client),
             )
             return _new_client
 
@@ -402,7 +408,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         data: dict,
         timeout: float | httpx.Timeout,
         logging_obj: LiteLLMLoggingObj,
-    ) -> Tuple[dict, BaseModel]:
+    ) -> tuple[dict, BaseModel]:
         """
         Helper to:
         - call chat.completions.create.with_raw_response when litellm.return_response_headers is True
@@ -439,7 +445,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         data: dict,
         timeout: float | httpx.Timeout,
         logging_obj: LiteLLMLoggingObj,
-    ) -> Tuple[dict, BaseModel]:
+    ) -> tuple[dict, BaseModel]:
         """
         Helper to:
         - call chat.completions.create.with_raw_response when litellm.return_response_headers is True
@@ -474,11 +480,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         self,
         response: Any,
         model: str,
-        messages: list[Dict],
-        optional_params: Dict,
+        messages: list[dict],
+        optional_params: dict,
         logging_obj: LiteLLMLoggingObj,
         stream: bool,
-        litellm_params: Dict,
+        litellm_params: dict,
     ) -> Any | None:
         """
         Call agentic completion hooks for all custom loggers (OpenAI Chat Completions API).
@@ -1288,7 +1294,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             )
 
             ## embedding CALL
-            headers: Dict | None = None
+            headers: dict | None = None
             headers, sync_embedding_response = self.make_sync_openai_embedding_request(
                 openai_client=openai_client,
                 data=data,
@@ -2842,7 +2848,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,
@@ -2881,12 +2887,12 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         tools: Iterable[AssistantToolParam] | None,
         event_handler: AssistantEventHandler | None,
     ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]:
-        data: Final[Dict[str, Any]] = {
+        data: Final[dict[str, Any]] = {
             "thread_id": thread_id,
             "assistant_id": assistant_id,
             "additional_instructions": additional_instructions,
@@ -2906,12 +2912,12 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         tools: Iterable[AssistantToolParam] | None,
         event_handler: AssistantEventHandler | None,
     ) -> AssistantStreamManager[AssistantEventHandler]:
-        data: Final[Dict[str, Any]] = {
+        data: Final[dict[str, Any]] = {
             "thread_id": thread_id,
             "assistant_id": assistant_id,
             "additional_instructions": additional_instructions,
@@ -2933,7 +2939,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,
@@ -2955,7 +2961,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,
@@ -2978,7 +2984,7 @@ class OpenAIAssistantsAPI(BaseLLM):
         assistant_id: str,
         additional_instructions: str | None,
         instructions: str | None,
-        metadata: Dict | None,
+        metadata: dict | None,
         model: str | None,
         stream: bool | None,
         tools: Iterable[AssistantToolParam] | None,

litellm/llms/openai/responses/transformation.py

@@ -165,10 +165,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         self,
         model: str,  # allows overrides to selectively run this
         input: str | ResponseInputParam,
-        tools: List[ALL_RESPONSES_API_TOOL_PARAMS] | None = None,
-    ) -> Tuple[
+        tools: list[ALL_RESPONSES_API_TOOL_PARAMS] | None = None,
+    ) -> tuple[
         str | ResponseInputParam,
-        List[ALL_RESPONSES_API_TOOL_PARAMS] | None,
+        list[ALL_RESPONSES_API_TOOL_PARAMS] | None,
     ]:
         """Sibling of `remove_cache_control_flag_from_messages_and_tools` on
         the chat path. Strips Anthropic-only `cache_control` markers from
@@ -447,7 +447,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> Tuple[str, dict]:
+    ) -> tuple[str, dict]:
         """
         Transform the delete response API request into a URL and data
 
@@ -482,7 +482,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> Tuple[str, dict]:
+    ) -> tuple[str, dict]:
         """
         Transform the get response API request into a URL and data
 
@@ -525,10 +525,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         headers: dict,
         after: str | None = None,
         before: str | None = None,
-        include: List[str] | None = None,
+        include: list[str] | None = None,
         limit: int = 20,
         order: Literal["asc", "desc"] = "desc",
-    ) -> Tuple[str, dict]:
+    ) -> tuple[str, dict]:
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}/input_items"
         params: Final[dict[str, Any]] = {}
@@ -563,7 +563,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> Tuple[str, dict]:
+    ) -> tuple[str, dict]:
         """
         Transform the cancel response API request into a URL and data
 
@@ -607,7 +607,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> Tuple[str, dict]:
+    ) -> tuple[str, dict]:
         """
         Transform the compact response API request into a URL and data
 

litellm/llms/openai/transcriptions/whisper_transformation.py

@@ -132,7 +132,7 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
                 raise
             return TranscriptionResponse(text=raw_response.text)
 
-        if any(key in raw_response_json for key in TranscriptionResponse.model_fields.keys()):
+        if any(key in raw_response_json for key in TranscriptionResponse.model_fields):
             return TranscriptionResponse(**raw_response_json)
         else:
             raise ValueError(

litellm/llms/sap/chat/models.py

@@ -1,6 +1,6 @@
 import warnings
 from enum import Enum
-from typing import Final, Literal, Union
+from typing import Final, Literal
 
 from pydantic import BaseModel, Field, field_validator, model_validator
 
@@ -115,7 +115,7 @@ class SAPToolChatMessage(BaseModel):
     _content_validator = field_validator("content", mode="before")(validate_different_content)
 
 
-ChatMessage = Union[SAPMessage, SAPUserMessage, SAPAssistantMessage, SAPToolChatMessage]
+ChatMessage = SAPMessage | SAPUserMessage | SAPAssistantMessage | SAPToolChatMessage
 
 
 class ResponseFormat(BaseModel):

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -11,8 +11,6 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
 import httpx
 
 import litellm
-import litellm.litellm_core_utils
-import litellm.litellm_core_utils.litellm_logging
 from litellm import verbose_logger
 from litellm._uuid import uuid
 from litellm.constants import (
@@ -2429,7 +2427,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         _candidates: Final = completion_response.get("candidates")
         if _candidates and len(_candidates) > 0:
             content_policy_violations: Final = VertexGeminiConfig().get_flagged_finish_reasons()
-            if "finishReason" in _candidates[0] and _candidates[0]["finishReason"] in content_policy_violations.keys():
+            if "finishReason" in _candidates[0] and _candidates[0]["finishReason"] in content_policy_violations:
                 return self._handle_content_policy_violation(
                     model_response=model_response,
                     completion_response=completion_response,

litellm/llms/volcengine/responses/transformation.py

@@ -1,4 +1,5 @@
 from collections.abc import Callable, Mapping, Sequence
+from types import UnionType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin
 
 import httpx
@@ -475,14 +476,12 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
             return 0
         if annotation is list or origin is list:
             return []
-        if origin is Union:
+        if origin is Union or origin is UnionType:
             # Prefer empty list when any option is a list
             if any((arg is list or VolcEngineResponsesAPIConfig._annotation_origin(arg) is list) for arg in args):
                 return []
             if type(None) in args:
                 return None
-        if origin is Union and type(None) in args:
-            return None
 
         # Fallback to None when no safer guess exists
         return None
@@ -514,7 +513,9 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
         Choose the best-matching Pydantic model class for a nested dict.
         """
         origin: Final = VolcEngineResponsesAPIConfig._annotation_origin(annotation)
-        union_args: Final = VolcEngineResponsesAPIConfig._annotation_args(annotation) if origin is Union else ()
+        union_args: Final = (
+            VolcEngineResponsesAPIConfig._annotation_args(annotation) if origin is Union or origin is UnionType else ()
+        )
         candidates = tuple(candidate for candidate in (annotation, *union_args) if hasattr(candidate, "model_fields"))
 
         if not candidates:

litellm/main.py

@@ -322,7 +322,7 @@ oci_transformation: Final = OCIChatConfig()
 ovhcloud_transformation: Final = OVHCloudChatConfig()
 lemonade_transformation: Final = LemonadeChatConfig()
 
-MOCK_RESPONSE_TYPE = Union[str, Exception, dict, ModelResponse, ModelResponseStream]
+MOCK_RESPONSE_TYPE = str | Exception | dict | ModelResponse | ModelResponseStream
 ####### COMPLETION ENDPOINTS ################
 
 
@@ -1174,13 +1174,14 @@ def _register_custom_pricing_for_request(
     shared_key: Final = f"{custom_llm_provider}/{model}"
     deployment_id: Final = _get_router_deployment_id(kwargs)
     if deployment_id is None:
-        litellm.register_model({shared_key: entry})
+        litellm.register_model({shared_key: entry}, persist_across_reloads=False)
         return
     litellm.register_model(
         {
             deployment_id: entry,
             shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry),
-        }
+        },
+        persist_across_reloads=False,
     )
 
 

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -3375,10 +3375,10 @@ class MCPServerManager:
 
                 static_headers: Final = server.static_headers or {}
                 has_static_authorization: Final = any(
-                    isinstance(k, str) and k.lower() == "authorization" for k in static_headers.keys()
+                    isinstance(k, str) and k.lower() == "authorization" for k in static_headers
                 )
                 has_extra_authorization: Final = bool(extra_headers) and any(
-                    isinstance(k, str) and k.lower() == "authorization" for k in (extra_headers or {}).keys()
+                    isinstance(k, str) and k.lower() == "authorization" for k in (extra_headers or {})
                 )
 
                 if (
@@ -4419,7 +4419,7 @@ class MCPServerManager:
         allowed_params_list: Final = allowed_params[matched]
 
         # Filter arguments to only include allowed parameters
-        disallowed_params: Final = [param for param in arguments.keys() if param not in allowed_params_list]
+        disallowed_params: Final = [param for param in arguments if param not in allowed_params_list]
 
         if disallowed_params:
             raise HTTPException(

litellm/proxy/_experimental/mcp_server/server.py

@@ -1613,7 +1613,7 @@ if MCP_AVAILABLE:
         ``mcp_server_auth_headers``). Either form skips the pre-emptive 401.
         """
         if oauth2_headers:
-            for k in oauth2_headers.keys():
+            for k in oauth2_headers:
                 if k.lower() == "authorization":
                     return True
         return _client_has_per_server_auth_header(server, mcp_server_auth_headers)

litellm/proxy/_types.py

@@ -3,7 +3,7 @@ import json
 import os
 from collections.abc import Callable
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Literal, Union
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
 from pydantic import (
@@ -67,7 +67,7 @@ from .types_utils.utils import get_instance_fn, validate_custom_validate_return_
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 
@@ -622,6 +622,8 @@ class LiteLLMRoutes(enum.Enum):
             "/team/permissions_update",
             "/team/permissions_bulk_update",
             "/team/daily/activity",
+            # gateway request counts (SGR); deployment-wide, admin-only
+            "/gateway/daily/activity",
             # model
             "/model/new",
             "/model/update",
@@ -715,6 +717,7 @@ class LiteLLMRoutes(enum.Enum):
         "/global/spend/tags",
         "/global/predict/spend/logs",
         "/global/activity",
+        "/gateway/daily/activity",
         "/health/services",
     ] + info_routes
 
@@ -2429,6 +2432,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="Maximum retention period for spend logs (e.g., '7d' for 7 days). Logs older than this will be deleted.",
     )
+    maximum_autorouter_session_retention_period: str | None = Field(
+        None,
+        description="Maximum retention period for auto-router benchmark session rollup rows (e.g., '365d'). Rows whose last turn is older than this are deleted by the spend log cleanup job, on that job's schedule. Unset means rollup rows are never deleted.",
+    )
     use_spend_logs_partitioning: bool | None = Field(
         None,
         description="If True and LiteLLM_SpendLogs has been converted to a range-partitioned table (db_scripts/partition_spend_logs.sql), retention cleanup drops expired partitions instead of deleting rows, and pre-creates upcoming partitions. Default is False.",
@@ -4010,7 +4017,7 @@ class JWTKeyItem(TypedDict, total=False):
     kid: str
 
 
-JWKKeyValue = Union[list[JWTKeyItem], JWTKeyItem]
+JWKKeyValue = list[JWTKeyItem] | JWTKeyItem
 
 
 class JWKUrlResponse(TypedDict, total=False):
@@ -4053,15 +4060,15 @@ class UserManagementEndpointParamDocStringEnums(str, enum.Enum):
     duration_doc_str = """Optional[str] - Duration for the key auto-created on `/user/new`. Default is None."""
 
 
-PassThroughEndpointLoggingResultValues = Union[
-    ModelResponse,
-    TextCompletionResponse,
-    ImageResponse,
-    EmbeddingResponse,
-    VideoObject,
-    StandardPassThroughResponseObject,
-    ResponsesAPIResponse,
-]
+PassThroughEndpointLoggingResultValues = (
+    ModelResponse
+    | TextCompletionResponse
+    | ImageResponse
+    | EmbeddingResponse
+    | VideoObject
+    | StandardPassThroughResponseObject
+    | ResponsesAPIResponse
+)
 
 
 class PassThroughEndpointLoggingTypedDict(TypedDict):
@@ -4162,7 +4169,7 @@ class ClientSideFallbackModel(TypedDict, total=False):
     messages: list[AllMessageValues]
 
 
-ALL_FALLBACK_MODEL_VALUES = Union[str, ClientSideFallbackModel]
+ALL_FALLBACK_MODEL_VALUES = str | ClientSideFallbackModel
 
 
 RBAC_ROLES = Literal[

litellm/proxy/a2a/version_convert.py

@@ -26,7 +26,7 @@ The two wire shapes:
 
 from collections.abc import Callable
 from types import ModuleType
-from typing import Final, Literal, Union
+from typing import Final, Literal
 
 from pydantic import BaseModel
 
@@ -34,7 +34,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.proxy.a2a.agent_card import normalize_protocol_version
 
 A2AVersion = Literal["0.3", "1.0"]
-RequestId = Union[str, int, None]
+RequestId = str | int | None
 JsonDict = dict[str, object]
 
 _V1_SEND_ENVELOPE_KEYS: Final = frozenset({"message", "task"})

litellm/proxy/auth/auth_checks.py

@@ -13,7 +13,7 @@ import asyncio
 import math
 import re
 import time
-from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
 
 from fastapi import HTTPException, Request, status
 from pydantic import BaseModel
@@ -109,7 +109,7 @@ from .auth_utils import get_model_from_request, get_request_route_template
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/proxy/auth/auth_exception_handler.py

@@ -2,7 +2,7 @@
 Handles Authentication Errors
 """
 
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, Final
 
 from fastapi import HTTPException, Request, status
 
@@ -28,7 +28,7 @@ DB_UNAVAILABLE_FALLBACK_USER_ID: Final = "__db_unavailable_fallback__"
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
 else:
     Span = Any
 

litellm/proxy/auth/ip_address_utils.py

@@ -7,7 +7,7 @@ External callers (public IPs) only see servers with available_on_public_internet
 
 import ipaddress
 from dataclasses import dataclass
-from typing import Any, Final, Union
+from typing import Any, Final
 
 from fastapi import Request
 from pydantic import TypeAdapter, ValidationError
@@ -45,7 +45,7 @@ class _HopCount:
     value: int
 
 
-_HopCountSetting = Union[_HopCountUnset, _HopCountInvalid, _HopCount]
+_HopCountSetting = _HopCountUnset | _HopCountInvalid | _HopCount
 
 
 class IPAddressUtils:

litellm/proxy/auth/network.py

@@ -1,14 +1,14 @@
 from __future__ import annotations
 
 import ipaddress
-from typing import Any, Final, Union
+from typing import Any, Final
 
 from fastapi import Request
 from pydantic import BaseModel, Field
 
 from litellm._logging import verbose_proxy_logger
 
-TrustedProxyNetwork = Union[ipaddress.IPv4Network, ipaddress.IPv6Network]
+TrustedProxyNetwork = ipaddress.IPv4Network | ipaddress.IPv6Network
 
 
 class NetworkContext(BaseModel):

litellm/proxy/auth/user_api_key_auth.py

@@ -1192,9 +1192,12 @@ async def _user_api_key_auth_builder(
             from litellm.proxy.proxy_server import premium_user
 
             if premium_user is not True:
-                raise ValueError(
-                    "Oauth2 token validation is only available for premium users"
-                    + CommonProxyErrors.not_premium_user.value
+                raise ProxyException(
+                    message="Oauth2 token validation is only available for premium users. "
+                    + CommonProxyErrors.not_premium_user.value,
+                    type=ProxyErrorTypes.auth_error,
+                    param="premium_user",
+                    code=status.HTTP_403_FORBIDDEN,
                 )
 
             return await Oauth2Handler.check_oauth2_token(token=api_key)

litellm/proxy/batches_endpoints/endpoints.py

@@ -177,7 +177,7 @@ async def create_batch(
             }
 
         input_file_id: Final = _create_batch_data.get("input_file_id", None)
-        unified_file_id: Union[str, Literal[False]] = False
+        unified_file_id: str | Literal[False] = False
 
         model_from_file_id = None
         if input_file_id:

litellm/proxy/client/cli/commands/autoroute/config.py

@@ -1,4 +1,4 @@
-from typing import Final, Literal, Union
+from typing import Final, Literal
 
 from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter
 
@@ -56,7 +56,7 @@ class LLMClassifier(BaseModel):
     timeout_ms: int = 3000
 
 
-ClassifierChoice = Union[HeuristicClassifier, LLMClassifier]
+ClassifierChoice = HeuristicClassifier | LLMClassifier
 
 
 class NoSemanticMatching(BaseModel):
@@ -88,7 +88,7 @@ class SemanticMatching(BaseModel):
     keyword_tier_rules: tuple[KeywordTierRule, ...] = DEFAULT_KEYWORD_TIER_RULES
 
 
-SemanticMatchingChoice = Union[NoSemanticMatching, SemanticMatching]
+SemanticMatchingChoice = NoSemanticMatching | SemanticMatching
 
 
 class AutorouteConfig(BaseModel):

litellm/proxy/db/autorouter_session_rollup.py

@@ -0,0 +1,322 @@
+"""
+Per-session auto-router benchmarks rollup.
+
+At request time the spend writer builds one AutoRouterTurnTransaction per successful
+auto-routed request (a request whose metadata carries a routing_decision) and queues it
+on the prisma client. The spend-log flush job drains the queue into
+LiteLLM_AutoRouterSession with one conditional upsert per turn: the statement classifies
+the turn (same model, first visit, return to a model the session already used, out of
+order) against the row's own columns, so nothing is read before the write and concurrent
+pods compose. The benchmarks endpoint aggregates these rows and never touches
+LiteLLM_SpendLogs.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import hashlib
+import random
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from itertools import groupby
+from typing import TYPE_CHECKING, Final, NamedTuple
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES
+
+if TYPE_CHECKING:
+    from litellm.proxy._types import SpendLogsPayload
+    from litellm.proxy.utils import PrismaClient
+
+CACHE_TTL_5M_SECONDS: Final = 300
+CACHE_TTL_1H_SECONDS: Final = 3600
+
+
+@dataclass(frozen=True, slots=True)
+class AutoRouterTurnTransaction:
+    api_key: str
+    session_id: str
+    router_name: str
+    router_type: str
+    model: str
+    turn_at: datetime
+    total_tokens: int
+    spend: float
+    saved_spend: float
+    covered: bool
+    cache_hit: bool
+    cache_ttl_seconds: int | None
+    cache_touched: bool
+
+
+class TurnCacheFacts(NamedTuple):
+    """One statement of a turn's cache interaction, derived from its usage record.
+
+    ``touched`` is False only when telemetry positively shows the provider neither
+    read from nor wrote to the cache; absent telemetry reads as touched, which is
+    the conservative input for the per-model idle clock.
+    """
+
+    covered: bool
+    read_tokens: int
+    write_ttl_seconds: int | None
+    touched: bool
+
+
+def turn_cache_facts(usage_object: Mapping[str, object] | None) -> TurnCacheFacts:
+    from litellm.proxy.spend_tracking.savings import extract_cache_read_tokens
+
+    covered: Final = bool(usage_object)
+    read_tokens: Final = extract_cache_read_tokens(usage_object)
+    write_ttl_seconds: Final = _write_ttl_seconds(usage_object)
+    return TurnCacheFacts(
+        covered=covered,
+        read_tokens=read_tokens,
+        write_ttl_seconds=write_ttl_seconds,
+        touched=not covered or read_tokens > 0 or write_ttl_seconds is not None,
+    )
+
+
+def _turn_time_utc(start_time_iso: str) -> datetime | None:
+    try:
+        parsed: Final = datetime.fromisoformat(start_time_iso.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return parsed
+    return parsed.astimezone(timezone.utc).replace(tzinfo=None)
+
+
+def _write_ttl_seconds(usage_object: Mapping[str, object] | None) -> int | None:
+    """The TTL this turn's cache write used, or None when nothing was written.
+
+    Providers that report a TTL split do so under prompt_tokens_details; a write with no
+    split is the provider's default five-minute cache.
+    """
+    from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens
+
+    if not usage_object:
+        return None
+    details: Final = usage_object.get("prompt_tokens_details")
+    creation: Final = details.get("cache_creation_token_details") if isinstance(details, Mapping) else None
+    if isinstance(creation, Mapping):
+        if creation.get("ephemeral_1h_input_tokens"):
+            return CACHE_TTL_1H_SECONDS
+        if creation.get("ephemeral_5m_input_tokens"):
+            return CACHE_TTL_5M_SECONDS
+    if extract_cache_creation_tokens(usage_object) > 0:
+        return CACHE_TTL_5M_SECONDS
+    return None
+
+
+SESSION_ID_MAX_CHARS: Final = 256
+
+
+def _bounded_session_id(session_id: str) -> str:
+    """The session id as stored, bounded so a caller-chosen identifier cannot exceed
+    Postgres's B-tree index entry limit through the composite primary key. Oversized
+    ids map to a stable digest, so their turns still aggregate into one session."""
+    if len(session_id) <= SESSION_ID_MAX_CHARS:
+        return session_id
+    return "sha256:" + hashlib.sha256(session_id.encode("utf-8", errors="surrogatepass")).hexdigest()
+
+
+def build_autorouter_turn_transaction(
+    payload: SpendLogsPayload,
+    metadata: Mapping[str, object],
+    saved_spend: float,
+) -> AutoRouterTurnTransaction | None:
+    """One rollup transaction for a successful auto-routed turn, else None.
+
+    The routing_decision record is what says a request was auto-routed at all, so a
+    request without one (including the auto-router's own classifier sub-calls) never
+    reaches the rollup. Failed requests served nothing and are excluded. Cache facts
+    are derived from the payload's own usage record through the savings owner, never
+    handed in beside it.
+    """
+    if payload.get("status") != "success":
+        return None
+    routing_decision: Final = metadata.get("routing_decision")
+    if not isinstance(routing_decision, Mapping) or not routing_decision:
+        return None
+    router_name: Final = routing_decision.get("router_model_name") or payload.get("model_group")
+    api_key: Final = payload.get("api_key")
+    session_id: Final = payload.get("session_id")
+    model: Final = payload.get("model")
+    if not (isinstance(router_name, str) and router_name and api_key and session_id and model):
+        return None
+    turn_at: Final = _turn_time_utc(str(payload.get("startTime") or ""))
+    if turn_at is None:
+        return None
+    usage_object_raw: Final = metadata.get("usage_object")
+    cache: Final = turn_cache_facts(usage_object_raw if isinstance(usage_object_raw, Mapping) else None)
+    return AutoRouterTurnTransaction(
+        api_key=api_key,
+        session_id=_bounded_session_id(session_id),
+        router_name=router_name,
+        router_type=str(routing_decision.get("router_type") or "unknown"),
+        model=model,
+        turn_at=turn_at,
+        total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0),
+        spend=float(payload.get("spend") or 0.0),
+        saved_spend=saved_spend,
+        covered=cache.covered,
+        cache_hit=cache.read_tokens > 0,
+        cache_ttl_seconds=cache.write_ttl_seconds,
+        cache_touched=cache.touched,
+    )
+
+
+_UPSERT_PARAM_FIELDS: Final = tuple(field.name for field in dataclasses.fields(AutoRouterTurnTransaction))
+
+
+def _p(field_name: str) -> str:
+    """Positional placeholder for a transaction field, numbered by the dataclass's own
+    field order so the SQL and the argument tuple cannot disagree; typos fail at import."""
+    return f"${_UPSERT_PARAM_FIELDS.index(field_name) + 1}"
+
+
+_MODEL: Final = _p("model")
+_TURN_AT: Final = _p("turn_at")
+_COVERED: Final = _p("covered")
+_CACHE_HIT: Final = _p("cache_hit")
+_CACHE_TTL: Final = _p("cache_ttl_seconds")
+_TOUCHED: Final = _p("cache_touched")
+
+_IN_ORDER: Final = f"{_TURN_AT}::timestamp >= t.last_turn_at"
+_SAME: Final = f"{_IN_ORDER} AND t.last_model = {_MODEL}"
+_FIRST: Final = f"{_IN_ORDER} AND NOT t.models ? {_MODEL}"
+_RETURN: Final = f"{_IN_ORDER} AND t.models ? {_MODEL} AND t.last_model <> {_MODEL}"
+_RETURN_MISS: Final = (
+    f"{_RETURN} AND {_COVERED}::int = 1 AND {_CACHE_HIT}::int = 0 AND (t.models -> {_MODEL} ->> 'ttl') IS NOT NULL"
+)
+_IDLE_SECONDS: Final = f"EXTRACT(EPOCH FROM {_TURN_AT}::timestamp) - (t.models -> {_MODEL} ->> 'at')::float8"
+_CACHE_TOUCHED: Final = f"{_TOUCHED}::int = 1"
+
+UPSERT_AUTOROUTER_SESSION_SQL: Final = f"""
+INSERT INTO "LiteLLM_AutoRouterSession" AS t (
+    api_key, session_id, router_name, router_type, first_turn_at, last_turn_at,
+    last_model, models, turns, unordered_turns, covered_turns, cache_hits,
+    same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
+    return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
+    ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend
+)
+VALUES (
+    {_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
+    {_MODEL}, jsonb_build_object({_MODEL}, jsonb_build_object('at', EXTRACT(EPOCH FROM {_TURN_AT}::timestamp), 'ttl', {_CACHE_TTL}::int)),
+    1, 0, {_COVERED}::int, {_CACHE_HIT}::int,
+    0, 0, 1, {_CACHE_HIT}::int,
+    0, 0, 0, 0,
+    (CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_5M_SECONDS} THEN 1 ELSE 0 END),
+    (CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_1H_SECONDS} THEN 1 ELSE 0 END),
+    {_p("total_tokens")}::bigint, {_p("spend")}::float8, {_p("saved_spend")}::float8
+)
+ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
+    turns = t.turns + 1,
+    total_tokens = t.total_tokens + EXCLUDED.total_tokens,
+    spend = t.spend + EXCLUDED.spend,
+    saved_spend = t.saved_spend + EXCLUDED.saved_spend,
+    covered_turns = t.covered_turns + EXCLUDED.covered_turns,
+    cache_hits = t.cache_hits + EXCLUDED.cache_hits,
+    ttl_5m_turns = t.ttl_5m_turns + EXCLUDED.ttl_5m_turns,
+    ttl_1h_turns = t.ttl_1h_turns + EXCLUDED.ttl_1h_turns,
+    unordered_turns = t.unordered_turns + (CASE WHEN NOT ({_IN_ORDER}) THEN 1 ELSE 0 END),
+    same_model_turns = t.same_model_turns + (CASE WHEN {_SAME} THEN 1 ELSE 0 END),
+    same_model_hits = t.same_model_hits + (CASE WHEN {_SAME} AND {_CACHE_HIT}::int = 1 THEN 1 ELSE 0 END),
+    first_visit_turns = t.first_visit_turns + (CASE WHEN {_FIRST} THEN 1 ELSE 0 END),
+    first_visit_hits = t.first_visit_hits + (CASE WHEN {_FIRST} AND {_CACHE_HIT}::int = 1 THEN 1 ELSE 0 END),
+    return_turns = t.return_turns + (CASE WHEN {_RETURN} THEN 1 ELSE 0 END),
+    return_hits = t.return_hits + (CASE WHEN {_RETURN} AND {_CACHE_HIT}::int = 1 THEN 1 ELSE 0 END),
+    return_expired_misses = t.return_expired_misses
+        + (CASE WHEN {_RETURN_MISS} AND {_IDLE_SECONDS} > (t.models -> {_MODEL} ->> 'ttl')::float8 THEN 1 ELSE 0 END),
+    return_within_ttl_misses = t.return_within_ttl_misses
+        + (CASE WHEN {_RETURN_MISS} AND {_IDLE_SECONDS} <= (t.models -> {_MODEL} ->> 'ttl')::float8 THEN 1 ELSE 0 END),
+    models = t.models || jsonb_build_object({_MODEL}, jsonb_build_object(
+        'at', (CASE WHEN {_CACHE_TOUCHED}
+               THEN GREATEST(COALESCE((t.models -> {_MODEL} ->> 'at')::float8, 0), EXTRACT(EPOCH FROM {_TURN_AT}::timestamp))
+               ELSE COALESCE((t.models -> {_MODEL} ->> 'at')::float8, EXTRACT(EPOCH FROM {_TURN_AT}::timestamp)) END),
+        'ttl', (CASE WHEN {_IN_ORDER}
+                THEN COALESCE({_CACHE_TTL}::int, (t.models -> {_MODEL} ->> 'ttl')::int)
+                ELSE COALESCE((t.models -> {_MODEL} ->> 'ttl')::int, {_CACHE_TTL}::int) END)
+    )),
+    last_model = (CASE WHEN {_IN_ORDER} THEN {_MODEL} ELSE t.last_model END),
+    first_turn_at = LEAST(t.first_turn_at, EXCLUDED.first_turn_at),
+    last_turn_at = GREATEST(t.last_turn_at, EXCLUDED.last_turn_at)
+"""
+
+
+def _as_sql_param(value: str | float | bool | datetime | None) -> str | float | None:
+    if isinstance(value, bool):
+        return int(value)
+    if isinstance(value, datetime):
+        return value.isoformat()
+    return value
+
+
+def _upsert_params(transaction: AutoRouterTurnTransaction) -> tuple[str | float | None, ...]:
+    return tuple(_as_sql_param(getattr(transaction, name)) for name in _UPSERT_PARAM_FIELDS)
+
+
+async def _upsert_turn_with_retry(
+    prisma_client: PrismaClient,
+    transaction: AutoRouterTurnTransaction,
+    n_retry_times: int,
+) -> None:
+    for attempt in range(n_retry_times + 1):
+        try:
+            await prisma_client.db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction))
+        except DB_RETRY_SAFE_ERROR_TYPES:
+            if attempt >= n_retry_times:
+                raise
+            await asyncio.sleep(2**attempt + random.uniform(0, 1))
+        else:
+            return
+
+
+async def flush_autorouter_turn_transactions(
+    prisma_client: PrismaClient,
+    transactions: Sequence[AutoRouterTurnTransaction],
+    n_retry_times: int = 3,
+) -> None:
+    """Drain a queue batch into the rollup, one upsert per turn.
+
+    Statements run sequentially in per-session event order: a turn's classification
+    depends on the turns before it, and Postgres rejects one multi-row INSERT touching
+    the same key twice. Only ConnectError is retried, per statement, because it proves
+    that statement never reached the database. Any other failure drops the remaining
+    turns of THAT session only, with an error log, and the flush continues with the
+    next session: sessions are independent state machines, so one poisoned statement
+    must not discard unrelated sessions, and a repeated increment is worse than an
+    undercount. Callers must not add their own retry around this function.
+    """
+    if not transactions:
+        return
+    ordered: Final = sorted(
+        transactions,
+        key=lambda transaction: (
+            transaction.api_key,
+            transaction.session_id,
+            transaction.router_name,
+            transaction.turn_at,
+        ),
+    )
+    for session_key, session_group in groupby(
+        ordered,
+        key=lambda transaction: (transaction.api_key, transaction.session_id, transaction.router_name),
+    ):
+        session_turns = tuple(session_group)
+        for position, transaction in enumerate(session_turns):
+            try:
+                await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times)
+            except Exception as flush_err:  # noqa: BLE001  # a statement failure drops only its session's remainder by design
+                verbose_proxy_logger.error(
+                    "Spend tracking - auto-router session rollup flush failed for router %s; "
+                    "%s of %s turn transactions dropped for one session: %s",
+                    session_key[2],
+                    len(session_turns) - position,
+                    len(session_turns),
+                    flush_err,
+                )
+                break

litellm/proxy/db/db_spend_update_writer.py

@@ -54,7 +54,11 @@ from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
 from litellm.proxy.spend_tracking.compression_savings import (
     extract_compression_saved_tokens,
 )
-from litellm.proxy.spend_tracking.savings import compute_savings_spend
+from litellm.proxy.spend_tracking.savings import (
+    compute_savings_spend,
+    extract_cache_creation_tokens,
+    extract_cache_read_tokens,
+)
 from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
 
 if TYPE_CHECKING:
@@ -84,31 +88,6 @@ def _get_llm_router():
         return None
 
 
-def _extract_cache_read_tokens(usage_obj: dict) -> int:
-    """
-    Anthropic: top-level cache_read_input_tokens field.
-    OpenAI-compatible (moonshotai, openai, deepseek, etc.): prompt_tokens_details.cached_tokens.
-    """
-    explicit: Final = usage_obj.get("cache_read_input_tokens", 0) or 0
-    if explicit:
-        return int(explicit)
-    details: Final = usage_obj.get("prompt_tokens_details") or {}
-    return int(details.get("cached_tokens", 0) or 0)
-
-
-def _extract_cache_creation_tokens(usage_obj: dict) -> int:
-    """
-    Anthropic: top-level cache_creation_input_tokens field.
-    OpenAI-compatible (kimi-k2 etc.): prompt_tokens_details.cache_write_tokens
-    or prompt_tokens_details.cache_creation_tokens.
-    """
-    explicit: Final = usage_obj.get("cache_creation_input_tokens", 0) or 0
-    if explicit:
-        return int(explicit)
-    details: Final = usage_obj.get("prompt_tokens_details") or {}
-    return int(details.get("cache_write_tokens", 0) or details.get("cache_creation_tokens", 0) or 0)
-
-
 class DBSpendUpdateWriter:
     """
     Module responsible for
@@ -204,6 +183,10 @@ class DBSpendUpdateWriter:
                     prisma_client=prisma_client,
                     kwargs=kwargs,
                 )
+                await self._enqueue_autorouter_turn_transaction(
+                    payload=payload,
+                    prisma_client=prisma_client,
+                )
             else:
                 verbose_proxy_logger.debug(
                     "disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
@@ -275,6 +258,47 @@ class DBSpendUpdateWriter:
         except Exception as e:
             verbose_proxy_logger.debug("_enqueue_tool_usage_transaction error (non-blocking): %s", e)
 
+    async def _enqueue_autorouter_turn_transaction(
+        self,
+        payload: SpendLogsPayload,
+        prisma_client: "PrismaClient | None",
+    ) -> None:
+        try:
+            if prisma_client is None:
+                return
+            metadata_raw: Final = payload.get("metadata")
+            if not metadata_raw:
+                return
+            metadata: Final = json.loads(metadata_raw)
+            if not isinstance(metadata, dict) or not metadata.get("routing_decision"):
+                return
+            from litellm.proxy.db.autorouter_session_rollup import (
+                build_autorouter_turn_transaction,
+            )
+
+            usage_object_raw: Final = metadata.get("usage_object")
+            savings_spend: Final = compute_savings_spend(
+                model=payload.get("model"),
+                custom_llm_provider=payload.get("custom_llm_provider"),
+                compression_saved_tokens=0,
+                routing_decision=metadata.get("routing_decision"),
+                usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
+                model_id=payload.get("model_id"),
+                llm_router=_get_llm_router,
+                cost_breakdown=metadata.get("cost_breakdown"),
+            )
+            transaction: Final = build_autorouter_turn_transaction(
+                payload=payload,
+                metadata=metadata,
+                saved_spend=savings_spend.autorouter,
+            )
+            if transaction is None:
+                return
+            async with prisma_client._autorouter_turn_transactions_lock:
+                prisma_client.autorouter_turn_transactions.append(transaction)
+        except Exception as e:  # noqa: BLE001  # a metrics enqueue must never fail the spend write
+            verbose_proxy_logger.debug("_enqueue_autorouter_turn_transaction error (non-blocking): %s", e)
+
     def _enqueue_tool_registry_upsert(
         self,
         kwargs: dict | None,
@@ -1230,7 +1254,7 @@ class DBSpendUpdateWriter:
         if team_member_list_transactions is not None and len(team_member_list_transactions.keys()) > 0:
             # Track which team memberships will be updated for cache invalidation
             team_memberships_to_invalidate: Final[list[tuple[str, str]]] = []
-            for key in team_member_list_transactions.keys():
+            for key in team_member_list_transactions:
                 # key is "team_id::<value>::user_id::<value>"
                 team_id = key.split("::")[1]
                 user_id = key.split("::")[3]
@@ -1860,6 +1884,12 @@ class DBSpendUpdateWriter:
             )
             return None
 
+        # TODO: remove the successful_requests/failed_requests counters below once the
+        # admin UI has fully migrated to LiteLLM_DailyGatewayRequests, which is now the
+        # source of truth for SGR. This path derives the counts from spend-log metadata
+        # rather than from what the gateway answered, so the two intentionally disagree
+        # (see litellm/proxy/middleware/billable_request_metrics_middleware.py). The
+        # spend, token and per-entity columns written here stay either way.
         request_status: Final = prisma_client.get_request_status(payload)
         verbose_proxy_logger.debug("Logged request status: %s", request_status)
         _metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
@@ -1881,13 +1911,12 @@ class DBSpendUpdateWriter:
             if call_type:
                 endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
 
-            cache_read_input_tokens: Final = _extract_cache_read_tokens(usage_obj)
+            cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj)
             compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata)
             savings_spend: Final = compute_savings_spend(
                 model=payload.get("model", None),
                 custom_llm_provider=payload.get("custom_llm_provider", None),
                 compression_saved_tokens=compression_saved_tokens,
-                cache_read_input_tokens=cache_read_input_tokens,
                 routing_decision=_metadata.get("routing_decision"),
                 model_id=payload.get("model_id"),
                 llm_router=_get_llm_router,
@@ -1910,7 +1939,7 @@ class DBSpendUpdateWriter:
                 successful_requests=1 if request_status == "success" else 0,
                 failed_requests=1 if request_status != "success" else 0,
                 cache_read_input_tokens=cache_read_input_tokens,
-                cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
+                cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj),
                 compression_saved_tokens=compression_saved_tokens,
                 compression_savings_spend=savings_spend.compression,
                 prompt_caching_savings_spend=savings_spend.prompt_caching,

litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py

@@ -46,32 +46,37 @@ class SpendLogCleanup:
         self.pod_lock_manager = pod_lock_manager
         verbose_proxy_logger.info("SpendLogCleanup initialized with batch size: %s", self.batch_size)
 
-    def _should_delete_spend_logs(self) -> bool:
+    def _retention_seconds_for(self, setting_name: str) -> int | None:
         """
-        Determines if logs should be deleted based on the max retention period in settings.
+        Parse one retention setting into seconds, or None when unset or invalid.
         """
-        retention_setting = self.general_settings.get("maximum_spend_logs_retention_period")
-        verbose_proxy_logger.info("Checking retention setting: %s", retention_setting)
+        retention_setting = self.general_settings.get(setting_name)
+        verbose_proxy_logger.info("Checking %s: %s", setting_name, retention_setting)
 
         if retention_setting is None:
-            verbose_proxy_logger.info("No retention setting found")
-            return False
+            return None
 
         try:
             if isinstance(retention_setting, int):
                 verbose_proxy_logger.warning(
-                    "maximum_spend_logs_retention_period is an integer (%s); treating as days. Use a string like '3d' to be explicit.",
+                    "%s is an integer (%s); treating as days. Use a string like '3d' to be explicit.",
+                    setting_name,
                     retention_setting,
                 )
                 retention_setting = f"{retention_setting}d"
-            self.retention_seconds = duration_in_seconds(retention_setting)
-            verbose_proxy_logger.info("Retention period set to %s seconds", self.retention_seconds)
-            return True
+            retention_seconds: Final = duration_in_seconds(retention_setting)
         except ValueError as e:
-            verbose_proxy_logger.warning(
-                "Invalid maximum_spend_logs_retention_period value: %s, error: %s", retention_setting, e
-            )
-            return False
+            verbose_proxy_logger.warning("Invalid %s value: %s, error: %s", setting_name, retention_setting, e)
+            return None
+        verbose_proxy_logger.info("%s set to %s seconds", setting_name, retention_seconds)
+        return retention_seconds
+
+    def _should_delete_spend_logs(self) -> bool:
+        """
+        Determines if logs should be deleted based on the max retention period in settings.
+        """
+        self.retention_seconds = self._retention_seconds_for("maximum_spend_logs_retention_period")
+        return self.retention_seconds is not None
 
     async def _delete_old_rows_batched(
         self,
@@ -186,6 +191,15 @@ class SpendLogCleanup:
             time_column="start_time",
         )
 
+    async def _delete_old_autorouter_session_rows(self, prisma_client: PrismaClient, cutoff_date: datetime) -> int:
+        return await self._delete_old_rows_batched(
+            prisma_client,
+            cutoff_date,
+            table_name="LiteLLM_AutoRouterSession",
+            key_columns=("api_key", "session_id", "router_name"),
+            time_column="last_turn_at",
+        )
+
     async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None:
         """
         Main cleanup function. Deletes old spend logs in batches.
@@ -196,10 +210,14 @@ class SpendLogCleanup:
         try:
             verbose_proxy_logger.info("Cleanup job triggered at %s", datetime.now())
 
-            if not self._should_delete_spend_logs():
+            delete_spend_logs: Final = self._should_delete_spend_logs()
+            autorouter_retention_seconds: Final = self._retention_seconds_for(
+                "maximum_autorouter_session_retention_period"
+            )
+            if not delete_spend_logs and autorouter_retention_seconds is None:
                 return
 
-            if self.retention_seconds is None:
+            if delete_spend_logs and self.retention_seconds is None:
                 verbose_proxy_logger.error("Retention seconds is None, cannot proceed with cleanup")
                 return
 
@@ -219,31 +237,41 @@ class SpendLogCleanup:
                     verbose_proxy_logger.info("Another pod is already running cleanup")
                     return
 
-            cutoff_date: Final = datetime.now(timezone.utc) - timedelta(seconds=float(self.retention_seconds))
-            verbose_proxy_logger.info("Removing logs older than %s", cutoff_date.isoformat())
+            if delete_spend_logs and self.retention_seconds is not None:
+                cutoff_date: Final = datetime.now(timezone.utc) - timedelta(seconds=float(self.retention_seconds))
+                verbose_proxy_logger.info("Removing logs older than %s", cutoff_date.isoformat())
 
-            if self.general_settings.get(
-                "use_spend_logs_partitioning", False
-            ) and await self.partition_manager.is_partitioned(prisma_client):
-                await self.partition_manager.ensure_partitions(prisma_client)
-                dropped: Final = await self.partition_manager.drop_partitions_older_than(prisma_client, cutoff_date)
-                verbose_proxy_logger.info(
-                    "Dropped %d expired spend-log partitions: %s",
-                    len(dropped),
-                    dropped,
+                if self.general_settings.get(
+                    "use_spend_logs_partitioning", False
+                ) and await self.partition_manager.is_partitioned(prisma_client):
+                    await self.partition_manager.ensure_partitions(prisma_client)
+                    dropped: Final = await self.partition_manager.drop_partitions_older_than(prisma_client, cutoff_date)
+                    verbose_proxy_logger.info(
+                        "Dropped %d expired spend-log partitions: %s",
+                        len(dropped),
+                        dropped,
+                    )
+                    # DROP only reclaims whole expired partitions. Expired rows can
+                    # still sit in the DEFAULT partition (backfill, coverage gaps)
+                    # or in a partition that spans the cutoff, so retention must
+                    # also delete those stragglers row-wise.
+                    total_deleted = await self._delete_old_logs(prisma_client, cutoff_date)
+                    verbose_proxy_logger.info(
+                        "Deleted %s expired logs not covered by dropped partitions", total_deleted
+                    )
+                else:
+                    total_deleted = await self._delete_old_logs(prisma_client, cutoff_date)
+                    verbose_proxy_logger.info("Deleted %s logs", total_deleted)
+
+                index_deleted: Final = await self._delete_old_tool_index_rows(prisma_client, cutoff_date)
+                verbose_proxy_logger.info("Deleted %s expired tool index rows", index_deleted)
+
+            if autorouter_retention_seconds is not None:
+                session_cutoff: Final = datetime.now(timezone.utc) - timedelta(
+                    seconds=float(autorouter_retention_seconds)
                 )
-                # DROP only reclaims whole expired partitions. Expired rows can
-                # still sit in the DEFAULT partition (backfill, coverage gaps)
-                # or in a partition that spans the cutoff, so retention must
-                # also delete those stragglers row-wise.
-                total_deleted = await self._delete_old_logs(prisma_client, cutoff_date)
-                verbose_proxy_logger.info("Deleted %s expired logs not covered by dropped partitions", total_deleted)
-            else:
-                total_deleted = await self._delete_old_logs(prisma_client, cutoff_date)
-                verbose_proxy_logger.info("Deleted %s logs", total_deleted)
-
-            index_deleted: Final = await self._delete_old_tool_index_rows(prisma_client, cutoff_date)
-            verbose_proxy_logger.info("Deleted %s expired tool index rows", index_deleted)
+                sessions_deleted: Final = await self._delete_old_autorouter_session_rows(prisma_client, session_cutoff)
+                verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_deleted)
 
         except Exception as e:
             # .exception() captures the traceback; str(e) alone on a Prisma/DB

litellm/proxy/db/gateway_request_tracking.py

@@ -0,0 +1,133 @@
+"""
+Accumulates gateway request counts (SGR) recorded at the ASGI edge and commits
+them to ``LiteLLM_DailyGatewayRequests``.
+
+Unlike the spend queues this keeps no per-request item. A count is a pure
+aggregate, so requests fold into an in-memory map as they finish. Every
+dimension of the key is server-chosen and drawn from a fixed set: the date, the
+category, and a route that the classifier maps to one of a closed list of
+strings rather than passing the raw path through. Nothing a caller sends can
+add a key, so the fold and the table it commits to are bounded by (days x
+routes) however much traffic arrives, and the response path carries no
+unbounded queue that would block once full.
+"""
+
+from dataclasses import asdict
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Final
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy.middleware.billable_request_metrics_middleware import BillableCategory
+from litellm.types.proxy.gateway_requests import (
+    GatewayRequestCounts,
+    GatewayRequestKey,
+    GatewayRequestSnapshot,
+)
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+_EMPTY: Final = GatewayRequestCounts(successful_requests=0, failed_requests=0)
+
+
+def _utc_date() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+
+class GatewayRequestAccumulator:
+    """Sink for the request-metrics middleware. ``record`` is sync and never awaits."""
+
+    def __init__(self) -> None:
+        self._counts: dict[GatewayRequestKey, GatewayRequestCounts] = {}  # mutable-ok: bounded fold, drained per flush
+
+    def record(self, *, category: BillableCategory, route: str, status_code: int) -> None:
+        key: Final = GatewayRequestKey(date=_utc_date(), category=category.value, route=route)
+        self._counts[key] = self._counts.get(key, _EMPTY).plus(succeeded=200 <= status_code < 300)
+
+    def drain(self) -> GatewayRequestSnapshot:
+        drained: Final = self._counts
+        self._counts = {}  # mutable-ok: the fold restarts empty; the drained map is handed off whole
+        return drained
+
+    def restore(self, snapshot: GatewayRequestSnapshot) -> None:
+        """
+        Merge un-committed counts back so the next flush retries them.
+
+        A dropped flush would silently undercount the metric the dashboard now
+        treats as the source of truth. Merging cannot grow without bound: keys
+        collapse on collision, so the fold stays bounded by (date x category x
+        route) however long the database is unreachable.
+
+        This buys at-least-once, not exactly-once, and the cost is worth stating.
+        The batch commits inside its context manager's ``__aexit__``, so a failure
+        raised after the transaction committed (a connection dropped while reading
+        the acknowledgement) restores counts that are already persisted, and the
+        next flush increments them a second time. Exactly-once would need a dedup
+        key the upserts could ignore on replay. For a traffic-volume metric a rare
+        overcount on a dropped acknowledgement beats losing a whole interval to
+        every database blip, so the trade is deliberate.
+        """
+        for key, counts in snapshot.items():
+            existing = self._counts.get(key, _EMPTY)
+            self._counts[key] = GatewayRequestCounts(
+                successful_requests=existing.successful_requests + counts.successful_requests,
+                failed_requests=existing.failed_requests + counts.failed_requests,
+            )
+
+
+async def commit_gateway_requests_to_db(
+    *,
+    prisma_client: "PrismaClient",
+    snapshot: GatewayRequestSnapshot,
+) -> None:
+    """Upsert one incrementing row per (date, category, route)."""
+    if not snapshot:
+        return
+
+    ordered: Final = sorted(snapshot.items(), key=lambda item: (item[0].date, item[0].category, item[0].route))
+
+    # pyright: ignore[reportAny] on both lines -- prisma's generated client is untyped,
+    # so .db and every table action off it resolve to Any at this boundary. The dict
+    # literals below are the shape prisma's generated inputs require.
+    async with prisma_client.db.batch_() as batcher:  # pyright: ignore[reportAny]  # untyped prisma client
+        for key, counts in ordered:
+            columns = asdict(key)
+            batcher.litellm_dailygatewayrequests.upsert(  # pyright: ignore[reportAny]  # untyped prisma client
+                where={"date_category_route": columns},  # mutable-ok: prisma input is dict-shaped
+                data={  # mutable-ok: prisma input is dict-shaped
+                    "create": {  # mutable-ok: prisma input is dict-shaped
+                        **columns,
+                        "successful_requests": counts.successful_requests,
+                        "failed_requests": counts.failed_requests,
+                    },
+                    "update": {  # mutable-ok: prisma input is dict-shaped
+                        "successful_requests": {"increment": counts.successful_requests},  # mutable-ok: as above
+                        "failed_requests": {"increment": counts.failed_requests},  # mutable-ok: as above
+                    },
+                },
+            )
+
+    verbose_proxy_logger.debug("Gateway request tracking - committed %d aggregated rows", len(ordered))
+
+
+async def flush_gateway_requests(
+    prisma_client: "PrismaClient",
+    accumulator: GatewayRequestAccumulator,
+) -> None:
+    """
+    Scheduler entrypoint. Never raises: a metering failure must not kill the job.
+
+    ``CancelledError`` is deliberately not caught, so a flush cancelled during
+    shutdown drops its snapshot rather than restoring counts onto an accumulator
+    the process is about to discard.
+    """
+    snapshot: Final = accumulator.drain()
+    try:
+        await commit_gateway_requests_to_db(prisma_client=prisma_client, snapshot=snapshot)
+    except Exception:  # noqa: BLE001 -- a failed flush must not stop the scheduler
+        accumulator.restore(snapshot)
+        verbose_proxy_logger.warning(
+            "Gateway request tracking - failed to commit %d rows, retrying on the next flush",
+            len(snapshot),
+            exc_info=True,
+        )

litellm/proxy/enterprise_billing/billing_metrics.py

@@ -16,7 +16,7 @@ payload; the secret license key is never sent as an attribute or header.
 import os
 import tempfile
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Final, Optional, Union
+from typing import TYPE_CHECKING, Final, Optional
 
 from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
 from opentelemetry.metrics import Counter
@@ -53,7 +53,7 @@ _CA_CERT_FILENAME: Final = "ca.crt"
 METRIC_NAME: Final = "litellm.enterprise.billable_requests"
 METER_NAME: Final = "litellm.enterprise.billing"
 
-AttributeValue = Union[str, int]
+AttributeValue = str | int
 
 
 @dataclass(frozen=True, slots=True)

litellm/proxy/fine_tuning_endpoints/endpoints.py

@@ -132,7 +132,7 @@ async def create_fine_tuning_job(
         )
 
         ## CHECK IF MANAGED FILE ID
-        unified_file_id: Union[str, Literal[False]] = False
+        unified_file_id: str | Literal[False] = False
         training_file: Final = fine_tuning_request.training_file
         response: LiteLLMFineTuningJob | None = None
         if training_file:
@@ -269,7 +269,7 @@ async def retrieve_fine_tuning_job(
         custom_llm_provider = request_body.get("custom_llm_provider", None) or custom_llm_provider
 
         ## CHECK IF MANAGED FILE ID
-        unified_finetuning_job_id: Union[str, Literal[False]] = False
+        unified_finetuning_job_id: str | Literal[False] = False
         response: LiteLLMFineTuningJob | None = None
         if fine_tuning_job_id:
             unified_finetuning_job_id = _is_base64_encoded_unified_file_id(fine_tuning_job_id)
@@ -536,7 +536,7 @@ async def cancel_fine_tuning_job(
         custom_llm_provider: Final = request_body.get("custom_llm_provider", None)
 
         ## CHECK IF MANAGED FILE ID
-        unified_finetuning_job_id: Union[str, Literal[False]] = False
+        unified_finetuning_job_id: str | Literal[False] = False
         response: LiteLLMFineTuningJob | None = None
         if fine_tuning_job_id:
             unified_finetuning_job_id = _is_base64_encoded_unified_file_id(fine_tuning_job_id)

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -8,7 +8,8 @@ import json
 import os
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, Union, cast
+from types import UnionType
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, Union, cast, get_args, get_origin
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -1556,13 +1557,9 @@ def _get_field_type_from_annotation(field_annotation: Any) -> str:
     Convert a Python type annotation to a UI-friendly type string
     """
     # Handle Union types (like Optional[T])
-    if (
-        hasattr(field_annotation, "__origin__")
-        and field_annotation.__origin__ is Union
-        and hasattr(field_annotation, "__args__")
-    ):
+    if get_origin(field_annotation) is Union or get_origin(field_annotation) is UnionType:
         # For Optional[T], get the non-None type
-        args: Final = field_annotation.__args__
+        args: Final = get_args(field_annotation)
         non_none_args: Final = [arg for arg in args if arg is not type(None)]
         if non_none_args:
             field_annotation = non_none_args[0]
@@ -1689,13 +1686,9 @@ def _should_skip_optional_params(field_name: str, field_annotation: Any) -> bool
 
 def _unwrap_optional_type(field_annotation: Any) -> Any:
     """Unwrap Optional types to get the actual type."""
-    if (
-        hasattr(field_annotation, "__origin__")
-        and field_annotation.__origin__ is Union
-        and hasattr(field_annotation, "__args__")
-    ):
+    if get_origin(field_annotation) is Union or get_origin(field_annotation) is UnionType:
         # For Optional[BaseModel], get the non-None type
-        args: Final = field_annotation.__args__
+        args: Final = get_args(field_annotation)
         non_none_args: Final = [arg for arg in args if arg is not type(None)]
         if non_none_args:
             return non_none_args[0]

litellm/proxy/guardrails/guardrail_helpers.py

@@ -10,7 +10,7 @@ from litellm.types.guardrails import *
 sys.path.insert(0, os.path.abspath("../.."))  # Adds the parent directory to the system path
 
 
-def can_modify_guardrails(team_obj: Optional[LiteLLM_TeamTable]) -> bool:
+def can_modify_guardrails(team_obj: LiteLLM_TeamTable | None) -> bool:
     if team_obj is None:
         return True
 

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py

@@ -265,7 +265,7 @@ class GenericGuardrailAPI(CustomGuardrail):
         # Dynamically iterate through GenericGuardrailAPIMetadata fields
         # and extract matching fields from the source metadata
         # Fields in metadata are already prefixed with 'user_api_key_'
-        for field_name in GenericGuardrailAPIMetadata.__annotations__.keys():
+        for field_name in GenericGuardrailAPIMetadata.__annotations__:
             value = metadata_dict.get(field_name)
             if value is not None:
                 result_metadata[field_name] = value

litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py

@@ -1,5 +1,5 @@
 from collections.abc import AsyncGenerator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal, Union
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
 from fastapi import HTTPException
@@ -57,7 +57,7 @@ class ModelArmorAPIError(Exception):
 
 _SCANNED_CONTENT_KEYS: Final = frozenset({"text", "sanitizedText", "findings", "maliciousUriMatchedItems"})
 
-RedactablePayload = Union[dict, list, str, int, float, bool, None]
+RedactablePayload = dict | list | str | int | float | bool | None
 
 
 def _redact_scanned_content(payload: RedactablePayload, depth: int = 0) -> RedactablePayload:

litellm/proxy/guardrails/guardrail_hooks/noma/noma.py

@@ -16,7 +16,6 @@ from typing import (
     Any,
     Final,
     Literal,
-    Union,
 )
 from urllib.parse import urljoin
 
@@ -54,7 +53,7 @@ SENSITIVE_DATA_DETECTOR_KEYS: Final[list[str]] = ["sensitiveData", "dataDetector
 
 # Type aliases
 MessageRole = Literal["user", "assistant"]
-LLMResponse = Union[Any, ModelResponse, EmbeddingResponse, ImageResponse]
+LLMResponse = Any | ModelResponse | EmbeddingResponse | ImageResponse
 _LEGACY_NOMA_DEPRECATION_WARNED = False
 
 if TYPE_CHECKING:

litellm/proxy/guardrails/usage_endpoints.py

@@ -6,7 +6,7 @@ GET /guardrails/usage/overview, /guardrails/usage/detail/:id, /guardrails/usage/
 import json
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Final, Literal, Union, overload
+from typing import TYPE_CHECKING, Any, Final, Literal, overload
 
 from fastapi import APIRouter, Depends, Query
 from pydantic import BaseModel
@@ -31,8 +31,8 @@ if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient
     from litellm.types.guardrails import Guardrail
 
-    _DbOrConfigGuardrail = Union[prisma_models.LiteLLM_GuardrailsTable, Guardrail]
-    _DailyMetricsRow = Union[prisma_models.LiteLLM_DailyGuardrailMetrics, prisma_models.LiteLLM_DailyPolicyMetrics]
+    _DbOrConfigGuardrail = prisma_models.LiteLLM_GuardrailsTable | Guardrail
+    _DailyMetricsRow = prisma_models.LiteLLM_DailyGuardrailMetrics | prisma_models.LiteLLM_DailyPolicyMetrics
 
 router: Final = APIRouter()
 

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -7,7 +7,7 @@ import time
 import traceback
 from collections.abc import Iterable
 from datetime import datetime, timedelta
-from typing import Any, Final, Literal, TypedDict, Union, cast
+from typing import Any, Final, Literal, TypedDict, cast
 
 import fastapi
 from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
@@ -110,7 +110,7 @@ def get_callback_identifier(callback):
 
 
 router: Final = APIRouter()
-services = Union[
+services = (
     Literal[
         "slack_budget_alerts",
         "langfuse",
@@ -127,9 +127,9 @@ services = Union[
         "galileo",
         "newrelic",
         "sqs",
-    ],
-    str,
-]
+    ]
+    | str
+)
 
 
 @router.get(

litellm/proxy/hooks/batch_rate_limiter.py

@@ -19,7 +19,7 @@ Quick summary:
 
 import json
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Union
+from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn
 
 from fastapi import HTTPException
 from pydantic import BaseModel
@@ -61,7 +61,7 @@ if TYPE_CHECKING:
     from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
     from litellm.router import Router as _Router
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     InternalUsageCache = _InternalUsageCache
     Router = _Router
     ParallelRequestLimiter = _ParallelRequestLimiter

litellm/proxy/hooks/batch_redis_get.py

@@ -53,7 +53,7 @@ class _PROXY_BatchRedisRequests(CustomLogger):
 
             key_value_dict = {}
             in_memory_cache_exists = False
-            for key in cache.in_memory_cache.cache_dict.keys():
+            for key in cache.in_memory_cache.cache_dict:
                 if isinstance(key, str) and key.startswith(cache_key_name):
                     in_memory_cache_exists = True
 

litellm/proxy/hooks/litellm_skills/main.py

@@ -170,7 +170,7 @@ class SkillsInjectionHook(CustomLogger):
             skill_files = self.prompt_handler.extract_all_files(skill)
             if skill_files:
                 all_skill_files[skill.skill_id] = skill_files
-                for path in skill_files.keys():
+                for path in skill_files:
                     if path.endswith(".py"):
                         all_module_paths.append(path)
 
@@ -238,7 +238,7 @@ class SkillsInjectionHook(CustomLogger):
             if skill_files:
                 all_skill_files[skill.skill_id] = skill_files
                 # Collect Python module paths
-                for path in skill_files.keys():
+                for path in skill_files:
                     if path.endswith(".py"):
                         all_module_paths.append(path)
 

litellm/proxy/hooks/parallel_request_limiter.py

@@ -1,7 +1,7 @@
 import asyncio
 import sys
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Union
+from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn
 
 from pydantic import BaseModel
 from typing_extensions import TypedDict
@@ -26,7 +26,7 @@ if TYPE_CHECKING:
 
     from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     InternalUsageCache = _InternalUsageCache
 else:
     Span = Any

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -12,7 +12,7 @@ from collections.abc import Callable
 from contextvars import ContextVar
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, cast
 
 from litellm import DualCache
 from litellm._logging import verbose_proxy_logger
@@ -49,7 +49,7 @@ if TYPE_CHECKING:
     from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
     from litellm.types.caching import RedisPipelineIncrementOperation
 
-    Span = Union[_Span, Any]
+    Span = _Span | Any
     InternalUsageCache = _InternalUsageCache
 else:
     Span = Any

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -4,8 +4,12 @@ AUTO ROUTER MANAGEMENT ENDPOINTS
 POST /auto_router/test_routing - Route one prompt through an unsaved complexity-router config
 """
 
+from collections.abc import Sequence
+from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Annotated, Final
 
+from pydantic import BaseModel, TypeAdapter
+
 from litellm._logging import verbose_proxy_logger
 from litellm.exceptions import BudgetExceededError
 from litellm.proxy._types import (
@@ -25,18 +29,23 @@ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
 from litellm.repositories.team_repository import TeamRepository
 from litellm.router_strategy.complexity_router import ComplexityRouter
 from litellm.types.management_endpoints.auto_router_endpoints import (
+    AutoRouterBenchmarkGroup,
+    AutoRouterBenchmarksResponse,
+    AutoRouterBenchmarkTotals,
+    AutoRouterCacheBucket,
+    AutoRouterCacheStats,
     AutoRouterRoutingTestRequest,
     AutoRouterRoutingTestResponse,
     RequestComplexityRouterConfig,
 )
 
 if TYPE_CHECKING:
-    from fastapi import APIRouter, Depends, HTTPException, status
+    from fastapi import APIRouter, Depends, HTTPException, Query, status
 
     from litellm.router import Router
 else:
     try:
-        from fastapi import APIRouter, Depends, HTTPException, status
+        from fastapi import APIRouter, Depends, HTTPException, Query, status
     except ImportError:
         # fastapi is only required for proxy, not for SDK usage
         pass
@@ -246,3 +255,202 @@ async def preview_auto_router_routing(
         routed_model_configured=hook_response.model in frozenset(llm_router.get_model_names()),
         routing_decision=hook_response.routing_decision,
     )
+
+
+class _SessionAggRow(BaseModel):
+    router_name: str
+    router_type: str
+    sessions: int
+    turns: int
+    unordered_turns: int
+    covered_turns: int
+    cache_hits: int
+    same_model_turns: int
+    same_model_hits: int
+    first_visit_turns: int
+    first_visit_hits: int
+    return_turns: int
+    return_hits: int
+    return_expired_misses: int
+    return_within_ttl_misses: int
+    ttl_5m_turns: int
+    ttl_1h_turns: int
+    total_tokens: int
+    spend: float
+    saved_spend: float
+    session_seconds: float
+
+
+_SESSION_AGG_ROWS: Final = TypeAdapter(list[_SessionAggRow])
+
+_BENCHMARKS_SQL: Final = """
+SELECT
+    router_name,
+    router_type,
+    COUNT(*)::int AS sessions,
+    COALESCE(SUM(turns), 0)::int AS turns,
+    COALESCE(SUM(unordered_turns), 0)::int AS unordered_turns,
+    COALESCE(SUM(covered_turns), 0)::int AS covered_turns,
+    COALESCE(SUM(cache_hits), 0)::int AS cache_hits,
+    COALESCE(SUM(same_model_turns), 0)::int AS same_model_turns,
+    COALESCE(SUM(same_model_hits), 0)::int AS same_model_hits,
+    COALESCE(SUM(first_visit_turns), 0)::int AS first_visit_turns,
+    COALESCE(SUM(first_visit_hits), 0)::int AS first_visit_hits,
+    COALESCE(SUM(return_turns), 0)::int AS return_turns,
+    COALESCE(SUM(return_hits), 0)::int AS return_hits,
+    COALESCE(SUM(return_expired_misses), 0)::int AS return_expired_misses,
+    COALESCE(SUM(return_within_ttl_misses), 0)::int AS return_within_ttl_misses,
+    COALESCE(SUM(ttl_5m_turns), 0)::int AS ttl_5m_turns,
+    COALESCE(SUM(ttl_1h_turns), 0)::int AS ttl_1h_turns,
+    COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens,
+    COALESCE(SUM(spend), 0)::float8 AS spend,
+    COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
+    COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds
+FROM "LiteLLM_AutoRouterSession"
+WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp
+GROUP BY router_name, router_type
+ORDER BY SUM(spend) DESC
+"""
+
+
+def _parse_benchmark_day(value: str) -> datetime:
+    try:
+        parsed: Final = datetime.strptime(value, "%Y-%m-%d").replace(tzinfo=timezone.utc)
+    except ValueError:
+        raise HTTPException(status_code=400, detail=f"Invalid date format: {value}. Expected: 'YYYY-MM-DD'")
+    return parsed.replace(tzinfo=None)
+
+
+def _pct(numerator: float, denominator: float) -> float:
+    if denominator <= 0:
+        return 0.0
+    return round(100.0 * numerator / denominator, 1)
+
+
+def _cache_bucket(turns: int, hits: int) -> AutoRouterCacheBucket:
+    return AutoRouterCacheBucket(turns=turns, hits=hits, hit_rate_pct=_pct(hits, turns))
+
+
+def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals:
+    return_misses: Final = row.return_turns - row.return_hits
+    baseline_spend: Final = row.spend + row.saved_spend
+    sessions: Final = row.sessions
+    return AutoRouterBenchmarkTotals(
+        sessions=sessions,
+        turns=row.turns,
+        avg_turns_per_session=row.turns / sessions if sessions else 0.0,
+        avg_session_seconds=row.session_seconds / sessions if sessions else 0.0,
+        avg_tokens_per_session=row.total_tokens / sessions if sessions else 0.0,
+        spend=row.spend,
+        saved_spend=row.saved_spend,
+        baseline_spend=baseline_spend,
+        saved_pct=_pct(row.saved_spend, baseline_spend),
+        saved_per_session=row.saved_spend / sessions if sessions else 0.0,
+        cache=AutoRouterCacheStats(
+            coverage_pct=_pct(row.covered_turns, row.turns),
+            hit_rate_pct=_pct(row.cache_hits, row.covered_turns),
+            same_model=_cache_bucket(row.same_model_turns, row.same_model_hits),
+            first_visit=_cache_bucket(row.first_visit_turns, row.first_visit_hits),
+            return_to_tier=_cache_bucket(row.return_turns, row.return_hits),
+            unordered_turns=row.unordered_turns,
+            return_misses_expired=row.return_expired_misses,
+            return_misses_within_ttl=row.return_within_ttl_misses,
+            return_misses_unknown=max(return_misses - row.return_expired_misses - row.return_within_ttl_misses, 0),
+            ttl_5m_turns=row.ttl_5m_turns,
+            ttl_1h_turns=row.ttl_1h_turns,
+        ),
+    )
+
+
+def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
+    return _SessionAggRow(
+        router_name="",
+        router_type="",
+        sessions=sum(row.sessions for row in rows),
+        turns=sum(row.turns for row in rows),
+        unordered_turns=sum(row.unordered_turns for row in rows),
+        covered_turns=sum(row.covered_turns for row in rows),
+        cache_hits=sum(row.cache_hits for row in rows),
+        same_model_turns=sum(row.same_model_turns for row in rows),
+        same_model_hits=sum(row.same_model_hits for row in rows),
+        first_visit_turns=sum(row.first_visit_turns for row in rows),
+        first_visit_hits=sum(row.first_visit_hits for row in rows),
+        return_turns=sum(row.return_turns for row in rows),
+        return_hits=sum(row.return_hits for row in rows),
+        return_expired_misses=sum(row.return_expired_misses for row in rows),
+        return_within_ttl_misses=sum(row.return_within_ttl_misses for row in rows),
+        ttl_5m_turns=sum(row.ttl_5m_turns for row in rows),
+        ttl_1h_turns=sum(row.ttl_1h_turns for row in rows),
+        total_tokens=sum(row.total_tokens for row in rows),
+        spend=sum(row.spend for row in rows),
+        saved_spend=sum(row.saved_spend for row in rows),
+        session_seconds=sum(row.session_seconds for row in rows),
+    )
+
+
+@router.get(
+    "/auto_router/benchmarks",
+    tags=("auto router",),
+    dependencies=(Depends(user_api_key_auth),),
+    response_model=AutoRouterBenchmarksResponse,
+)
+async def get_auto_router_benchmarks(
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    start_date: Annotated[
+        str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to 30 days before end_date)")
+    ] = None,
+    end_date: Annotated[str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to today)")] = None,
+) -> AutoRouterBenchmarksResponse:
+    """
+    Benchmarks for the auto-router dashboard: session shape, savings against the configured
+    baseline, and prompt-caching behaviour bucketed by what the router did.
+
+    Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time,
+    so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it
+    overlaps it: its last turn is on or after start_date and its first turn is on or before
+    end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
+    over that bucket's turns.
+    """
+    from litellm.proxy.proxy_server import prisma_client
+
+    if user_api_key_dict.user_role not in (
+        LitellmUserRoles.PROXY_ADMIN,
+        LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+    ):
+        raise HTTPException(
+            status_code=403,
+            detail="Only proxy admin roles can view auto-router benchmarks across the deployment",
+        )
+    if prisma_client is None:
+        raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
+
+    end_day: Final = (
+        _parse_benchmark_day(end_date)
+        if end_date
+        else datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None)
+    )
+    start_day: Final = _parse_benchmark_day(start_date) if start_date else end_day - timedelta(days=30)
+    if end_day < start_day:
+        raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date")
+
+    raw_rows: Final = await prisma_client.db.query_raw(
+        _BENCHMARKS_SQL,
+        start_day.isoformat(),
+        (end_day + timedelta(days=1)).isoformat(),
+    )
+    rows: Final = _SESSION_AGG_ROWS.validate_python(raw_rows or ())
+    groups: Final = tuple(
+        AutoRouterBenchmarkGroup(
+            router_name=row.router_name,
+            router_type=row.router_type,
+            **_benchmark_totals(row).model_dump(),
+        )
+        for row in rows
+    )
+    return AutoRouterBenchmarksResponse(
+        start_date=start_day.strftime("%Y-%m-%d"),
+        end_date=end_day.strftime("%Y-%m-%d"),
+        routers_in_scope=len(rows),
+        totals=_benchmark_totals(_summed_agg_row(rows)),
+        groups=groups,
+    )

litellm/proxy/management_endpoints/cache_settings_endpoints.py

@@ -249,7 +249,7 @@ def _redact_settings(settings: Mapping[str, object] | None) -> dict[str, object]
     """
     if not settings:
         return {}
-    return {k: _REDACTED_VALUE for k in settings.keys()}
+    return {k: _REDACTED_VALUE for k in settings}
 
 
 def _log_audit_task_exception(task: "asyncio.Task[None]") -> None:

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -2,7 +2,7 @@ import asyncio
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from datetime import datetime
 from types import SimpleNamespace
-from typing import TYPE_CHECKING, Final, Protocol, Union
+from typing import TYPE_CHECKING, Final, Protocol
 
 from fastapi import HTTPException, status
 from typing_extensions import TypedDict
@@ -109,7 +109,7 @@ class _KeyMetadataDict(TypedDict, total=False):
     team_id: str | None
 
 
-_WhereValue = Union[str, dict[str, object]]
+_WhereValue = str | dict[str, object]
 
 
 class _AggregatedSpendData(TypedDict):
@@ -571,6 +571,11 @@ def _build_aggregated_sql_query(
     # straight into their buckets without re-summing. The leaf grouping
     # is omitted on purpose: nothing in the response shape needs it once
     # all the rollups are present.
+    #
+    # TODO: drop the successful_requests/failed_requests aggregates (and the
+    # total_successful_requests metadata they feed) once the admin UI reads SGR
+    # only from LiteLLM_DailyGatewayRequests. The remaining spend, token and
+    # api_requests rollups are still served from here.
     sql_query: Final = f"""
         SELECT
             date,

litellm/proxy/management_endpoints/config_override_endpoints.py

@@ -54,7 +54,7 @@ def _redact_config(config: Mapping[str, Any] | None) -> dict[str, Any]:
     """
     if not config:
         return {}
-    return {k: _AUDIT_REDACTED for k in config.keys()}
+    return {k: _AUDIT_REDACTED for k in config}
 
 
 def _log_audit_task_exception(task: "asyncio.Task[None]") -> None:

litellm/proxy/management_endpoints/customer_endpoints.py

@@ -365,7 +365,7 @@ async def new_end_user(
         _user_data: Final = data.dict(exclude_none=True)
 
         for k, v in _user_data.items():
-            if k not in BudgetNewRequest.model_fields.keys():
+            if k not in BudgetNewRequest.model_fields:
                 new_end_user_obj[k] = v
 
         ## Handle Object Permission - MCP Servers, Vector Stores etc.
@@ -573,10 +573,10 @@ async def update_end_user(
             # budget_id is for linking to existing budget, not for creating new budget
             if k == "budget_id":
                 update_end_user_table_data[k] = v
-            elif k in LiteLLM_BudgetTable.model_fields.keys():
+            elif k in LiteLLM_BudgetTable.model_fields:
                 budget_table_data[k] = v
 
-            elif k in LiteLLM_EndUserTable.model_fields.keys():
+            elif k in LiteLLM_EndUserTable.model_fields:
                 update_end_user_table_data[k] = v
 
         ## Handle object permission updates (MCP servers, vector stores, etc.)

litellm/proxy/management_endpoints/gateway_request_endpoints.py

@@ -0,0 +1,139 @@
+"""
+GATEWAY REQUEST COUNTS (SGR)
+
+GET /gateway/daily/activity - successful/failed gateway requests by date and route
+
+Source of truth is LiteLLM_DailyGatewayRequests, written at the ASGI edge by
+BillableRequestMetricsMiddleware. This counts what the proxy answered, so it is
+independent of whether a request reached litellm's logging callbacks.
+
+The table carries no key/user/team dimension, so these totals are deployment-wide
+and the endpoint is restricted to proxy admin roles.
+"""
+
+from collections.abc import Sequence
+from datetime import datetime, timedelta, timezone
+from typing import Annotated, Final
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel, TypeAdapter
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.types.proxy.gateway_requests import (
+    GatewayRequestActivityResponse,
+    GatewayRequestBreakdownEntry,
+    GatewayRequestDailyEntry,
+)
+
+router: Final = APIRouter()
+
+_DEFAULT_LOOKBACK_DAYS: Final = 30
+
+_AGGREGATE_SQL: Final = """
+    SELECT
+        date,
+        category,
+        route,
+        SUM(successful_requests)::bigint AS successful_requests,
+        SUM(failed_requests)::bigint AS failed_requests
+    FROM "LiteLLM_DailyGatewayRequests"
+    WHERE date >= $1 AND date <= $2
+    GROUP BY date, category, route
+"""
+
+
+class _AggregateRow(BaseModel):
+    """Validates one query_raw row so the handler works with typed values, not Any."""
+
+    date: str
+    category: str
+    route: str
+    successful_requests: int
+    failed_requests: int
+
+
+_ROWS_ADAPTER: Final = TypeAdapter(tuple[_AggregateRow, ...])
+
+
+def _default_range() -> tuple[str, str]:
+    end: Final = datetime.now(timezone.utc)
+    start: Final = end - timedelta(days=_DEFAULT_LOOKBACK_DAYS)
+    return start.strftime("%Y-%m-%d"), end.strftime("%Y-%m-%d")
+
+
+def _fold_by_date(rows: Sequence[_AggregateRow]) -> tuple[GatewayRequestDailyEntry, ...]:
+    dates: Final = sorted(frozenset(row.date for row in rows))
+    return tuple(
+        GatewayRequestDailyEntry(
+            date=date,
+            successful_requests=sum(row.successful_requests for row in rows if row.date == date),
+            failed_requests=sum(row.failed_requests for row in rows if row.date == date),
+        )
+        for date in dates
+    )
+
+
+def _fold_by_route(rows: Sequence[_AggregateRow]) -> tuple[GatewayRequestBreakdownEntry, ...]:
+    pairs: Final = sorted(frozenset((row.category, row.route) for row in rows))
+    entries: Final = tuple(
+        GatewayRequestBreakdownEntry(
+            category=category,
+            route=route,
+            successful_requests=sum(
+                row.successful_requests for row in rows if row.category == category and row.route == route
+            ),
+            failed_requests=sum(row.failed_requests for row in rows if row.category == category and row.route == route),
+        )
+        for category, route in pairs
+    )
+    return tuple(sorted(entries, key=lambda entry: entry.successful_requests, reverse=True))
+
+
+@router.get(
+    "/gateway/daily/activity",
+    tags=["Budget & Spend Tracking"],  # mutable-ok: fastapi's decorator signature types tags as a list
+    response_model=GatewayRequestActivityResponse,
+)
+async def get_gateway_daily_activity(
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+    start_date: str | None = Query(default=None, description="Start date in YYYY-MM-DD format"),
+    end_date: str | None = Query(default=None, description="End date in YYYY-MM-DD format"),
+) -> GatewayRequestActivityResponse:
+    """
+    Successful and failed gateway requests, counted at the ASGI edge.
+
+    Deployment-wide: the underlying table has no per-key or per-user dimension,
+    so this is admin-only.
+    """
+    from litellm.proxy.proxy_server import prisma_client
+
+    if user_api_key_dict.user_role not in (
+        LitellmUserRoles.PROXY_ADMIN,
+        LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
+    ):
+        raise HTTPException(
+            status_code=403,
+            detail="Only proxy admin roles can view gateway request counts across the deployment",
+        )
+
+    if prisma_client is None:
+        raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
+
+    default_start, default_end = _default_range()
+    raw_rows: Final = await prisma_client.db.query_raw(  # pyright: ignore[reportAny]  # untyped prisma client
+        _AGGREGATE_SQL,
+        start_date or default_start,
+        end_date or default_end,
+    )
+    # Every downstream use is typed: the adapter returns _AggregateRow or raises.
+    rows: Final = _ROWS_ADAPTER.validate_python(raw_rows or ())
+    verbose_proxy_logger.debug("/gateway/daily/activity - aggregated %d rows", len(rows))
+
+    return GatewayRequestActivityResponse(
+        total_successful_requests=sum(row.successful_requests for row in rows),
+        total_failed_requests=sum(row.failed_requests for row in rows),
+        by_date=_fold_by_date(rows),
+        by_route=_fold_by_route(rows),
+    )

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -584,7 +584,7 @@ async def new_user(
         special_keys: Final = ["token", "token_id"]
         response_dict: Final = {}
         for key, value in response.items():
-            if key in NewUserResponse.model_fields.keys() and key not in special_keys:
+            if key in NewUserResponse.model_fields and key not in special_keys:
                 response_dict[key] = value
 
         response_dict["key"] = response.get("token", "")

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -215,7 +215,7 @@ async def _check_custom_key_allowed(custom_key_value: str | None) -> None:
         )
 
 
-def _is_team_key(data: Union[GenerateKeyRequest, LiteLLM_VerificationToken]):
+def _is_team_key(data: GenerateKeyRequest | LiteLLM_VerificationToken):
     return data.team_id is not None
 
 
@@ -498,7 +498,7 @@ def key_generation_check(
 
 def common_key_access_checks(
     user_api_key_dict: UserAPIKeyAuth,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
     llm_router: Router | None,
     premium_user: bool,
     user_id: str | None = None,
@@ -752,7 +752,7 @@ _BUDGET_NUMERIC_KEYS = frozenset(["max_budget", "soft_budget", "max_parallel_req
 
 
 def _enforce_upperbound_key_params(
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
     fill_defaults: bool = True,
 ) -> None:
     """
@@ -1161,7 +1161,7 @@ async def _common_key_generation_helper(
 
 def _check_key_model_specific_limits(
     keys: list[LiteLLM_VerificationToken],
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
     entity_rpm_limit: int | None,
     entity_tpm_limit: int | None,
     entity_model_rpm_limit_dict: dict[str, int],
@@ -1232,7 +1232,7 @@ def _check_key_model_specific_limits(
 
 def _check_key_rpm_tpm_limits(
     keys: list[LiteLLM_VerificationToken],
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
     entity_rpm_limit: int | None,
     entity_tpm_limit: int | None,
     entity_type: str,  # "team" or "organization"
@@ -1271,7 +1271,7 @@ def _check_key_rpm_tpm_limits(
 def check_team_key_model_specific_limits(
     keys: list[LiteLLM_VerificationToken],
     team_table: LiteLLM_TeamTableCachedObj,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
     """
     Check if the team key is allocating model specific limits. If so, raise an error if we're overallocating.
@@ -1296,7 +1296,7 @@ def check_team_key_model_specific_limits(
 def check_team_key_rpm_tpm_limits(
     keys: list[LiteLLM_VerificationToken],
     team_table: LiteLLM_TeamTableCachedObj,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
     """
     Check if the team key is allocating rpm/tpm limits. If so, raise an error if we're overallocating.
@@ -1312,7 +1312,7 @@ def check_team_key_rpm_tpm_limits(
 
 async def _check_team_key_limits(
     team_table: LiteLLM_TeamTableCachedObj,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
     prisma_client: PrismaClient,
 ) -> None:
     """
@@ -1348,7 +1348,7 @@ async def _check_team_key_limits(
 
 async def _check_project_key_limits(
     project_id: str,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
     prisma_client: PrismaClient,
     user_api_key_cache: UserApiKeyCache,
 ) -> None:
@@ -1398,7 +1398,7 @@ async def _check_project_key_limits(
 def check_org_key_model_specific_limits(
     keys: list[LiteLLM_VerificationToken],
     org_table: LiteLLM_OrganizationTable,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
     """
     Check if the organization key is allocating model specific limits. If so, raise an error if we're overallocating.
@@ -1431,7 +1431,7 @@ def check_org_key_model_specific_limits(
 def check_org_key_rpm_tpm_limits(
     keys: list[LiteLLM_VerificationToken],
     org_table: LiteLLM_OrganizationTable,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
     """
     Check if the organization key is allocating rpm/tpm limits. If so, raise an error if we're overallocating.
@@ -1487,7 +1487,7 @@ async def _validate_caller_can_assign_key_org(
 
 async def _check_org_key_limits(
     org_table: LiteLLM_OrganizationTable,
-    data: Union[GenerateKeyRequest, UpdateKeyRequest],
+    data: GenerateKeyRequest | UpdateKeyRequest,
     prisma_client: PrismaClient,
 ) -> None:
     """
@@ -1944,7 +1944,7 @@ def prepare_metadata_fields(data: BaseModel, non_default_values: dict, existing_
 
 
 async def prepare_key_update_data(
-    data: Union[UpdateKeyRequest, RegenerateKeyRequest],
+    data: UpdateKeyRequest | RegenerateKeyRequest,
     existing_key_row: LiteLLM_VerificationToken,
 ):
     data_json: Final[dict] = data.model_dump(exclude_unset=True)
@@ -5672,7 +5672,7 @@ def _build_key_filter_conditions(
     agent_id: str | None = None,
     use_substring_matching: bool = False,
     expires_filter: str | None = None,
-) -> dict[str, Union[str, dict[str, Any], list[dict[str, Any]]]]:
+) -> dict[str, str | dict[str, Any] | list[dict[str, Any]]]:
     """Build filter conditions for key listing.
 
     Visibility rules:
@@ -5684,7 +5684,7 @@ def _build_key_filter_conditions(
       so former members cannot see service accounts they created after leaving.
     """
     # Prepare filter conditions
-    where: dict[str, Union[str, dict[str, Any], list[dict[str, Any]]]] = {}
+    where: dict[str, str | dict[str, Any] | list[dict[str, Any]]] = {}
     where.update(_get_condition_to_filter_out_ui_session_tokens())
 
     # Build the OR conditions for user's keys and admin team keys
@@ -5918,7 +5918,7 @@ async def _list_key_helper(
             user_map = {user.user_id: user for user in users}
 
     # Prepare response
-    key_list: Final[list[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]]] = []
+    key_list: Final[list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]] = []
     for key in keys:
         # Convert Prisma model to dict (supports both Pydantic v1 and v2)
         try:

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -14,10 +14,11 @@ import asyncio
 import datetime
 import json
 from collections.abc import Mapping, Sequence
+from json import JSONDecodeError
 from typing import Any, Final, Literal, cast
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -59,12 +60,20 @@ from litellm.repositories.model_repository import ModelRepository
 from litellm.repositories.table_repositories import ModelTableRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.router import Router
+from litellm.router_strategy.complexity_router import (
+    DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
+    ComplexityRouterConfig,
+    ComplexityTier,
+    canonical_rubric_entries,
+    classification_system_prompt,
+)
 from litellm.router_utils.auto_router_model_naming import (
     STRATEGY_ROUTER_PARAM_FIELDS,
     validate_complexity_router_config_write,
     validate_strategy_router_model_write,
 )
 from litellm.types.proxy.management_endpoints.model_management_endpoints import (
+    AutoRouterClassifierDefaultPromptResponse,
     UpdateUsefulLinksRequest,
 )
 from litellm.types.router import (
@@ -1760,6 +1769,70 @@ async def update_useful_links(
         )
 
 
+def _labeled_tiers_from_query(tier_labels: str | None) -> tuple[tuple[ComplexityTier, str], ...] | None:
+    """Resolve the tier_labels query param into the labeled tiers the rubric is built from.
+
+    Validated through ComplexityRouterConfig so the editor prefills what the router would send: the
+    same field validators that reject a blank, duplicated, or canonical-name-stealing label on the
+    write path reject it here, rather than this returning a rubric no router could be configured to
+    use. A malformed value is the caller's error, so it surfaces as a 400.
+
+    None when unset, letting classification_system_prompt apply its own default names.
+    """
+    if not tier_labels:
+        return None
+    try:
+        return ComplexityRouterConfig(tier_labels=json.loads(tier_labels)).labeled_tiers()
+    except (JSONDecodeError, ValidationError) as e:
+        raise ProxyException(
+            message=f"tier_labels must be a JSON object of tier name to display name: {e}",
+            type=ProxyErrorTypes.bad_request_error,
+            code=status.HTTP_400_BAD_REQUEST,
+            param="tier_labels",
+        ) from e
+
+
+@router.get(
+    "/auto_router/classifier/default_prompt",
+    description="Get the built-in system prompt used by an auto-router's LLM classifier",
+    tags=["model management"],  # mutable-ok: fastapi's decorator signature types tags as a list
+    dependencies=[Depends(user_api_key_auth)],  # mutable-ok: fastapi's decorator signature types dependencies as a list
+)
+async def get_auto_router_classifier_default_prompt(
+    context_window_size: int = DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
+    tier_labels: str | None = None,
+) -> AutoRouterClassifierDefaultPromptResponse:
+    """
+    Get the default classifier system prompt, so the dashboard's prompt editor can prefill it.
+
+    The prompt's closing line depends on whether prior conversation turns are quoted to the
+    classifier, and its tier bullets are named by the router's tier_labels, so the caller passes both
+    to get the text that router would actually send rather than a rubric it does not use.
+
+    Parameters:
+    - context_window_size: int - The router's classifier_context_window_size. Defaults to the
+      built-in default.
+    - tier_labels: str | None - The router's tier_labels as a JSON object of canonical tier name to
+      display name, e.g. `{"SIMPLE": "Cheap"}`. Omit or pass an empty object for the default names.
+    """
+    if context_window_size < 0:
+        raise ProxyException(
+            message="context_window_size must be non-negative",
+            type=ProxyErrorTypes.bad_request_error,
+            code=status.HTTP_400_BAD_REQUEST,
+            param="context_window_size",
+        )
+
+    labeled_tiers: Final = _labeled_tiers_from_query(tier_labels)
+    return AutoRouterClassifierDefaultPromptResponse(
+        system_prompt=(
+            classification_system_prompt(context_window_size)
+            if labeled_tiers is None
+            else classification_system_prompt(context_window_size, tier_entries=canonical_rubric_entries(labeled_tiers))
+        )
+    )
+
+
 def _deduplicate_litellm_router_models(models: list[dict]) -> list[dict]:
     """
     Deduplicate models based on their model_info.id field.

litellm/proxy/management_endpoints/organization_endpoints.py

@@ -696,7 +696,7 @@ async def update_organization(
 
     # Handle budget updates if budget fields are provided
     budget_fields: Final = {
-        k: v for k, v in data.model_dump().items() if k in LiteLLM_BudgetTable.model_fields.keys() and v is not None
+        k: v for k, v in data.model_dump().items() if k in LiteLLM_BudgetTable.model_fields and v is not None
     }
 
     if budget_fields and existing_organization_row.budget_id:
@@ -706,7 +706,7 @@ async def update_organization(
         )
 
     # Remove budget fields from organization update data
-    for field in LiteLLM_BudgetTable.model_fields.keys():
+    for field in LiteLLM_BudgetTable.model_fields:
         updated_organization_row.pop(field, None)
 
     response: Final = await _table(OrganizationRepository(prisma_client)).update(

litellm/proxy/middleware/billable_request_metrics_middleware.py

@@ -1,11 +1,18 @@
 """
-Counts billable HTTP requests on enterprise deployments.
+Counts HTTP requests to LLM inference, MCP, and A2A endpoints.
 
-A billable request is an inbound request to an LLM inference, MCP, or A2A
-endpoint that returns a 2xx status. The actual export happens in an injected
-recorder (see litellm.proxy.enterprise_billing.billing_metrics); when no
-recorder is injected (non-enterprise, or metering misconfigured) this
-middleware is a transparent pass-through.
+Feeds two independent sinks off one classification:
+
+- ``GatewayRequestSink`` receives every classified request with its status and
+  is the source of truth for SGR (successful gateway requests) on the admin UI.
+  Not license-gated (see litellm.proxy.db.gateway_request_tracking). It is not
+  told which deployment served the request: it persists its counts, so every
+  dimension it takes has to be one the proxy chooses.
+- ``BillingRecorder`` receives 2xx requests only and exports them for
+  enterprise metering (see litellm.proxy.enterprise_billing.billing_metrics).
+
+Both are injected. When neither is present the middleware is a transparent
+pass-through.
 """
 
 import re
@@ -31,6 +38,21 @@ class BillingRecorder(Protocol):
     def record(self, *, category: BillableCategory, route: str, status_code: int, model_id: str | None) -> None: ...
 
 
+@runtime_checkable
+class GatewayRequestSink(Protocol):
+    """
+    Records every classified request, 2xx or not, for the SGR dashboard.
+
+    Distinct from BillingRecorder on three counts: this is not license-gated,
+    it is not restricted to 2xx, and it takes no model id. The deployment that
+    served a request is deliberately not part of what it records, because the
+    dashboard aggregates by route and a per-deployment dimension would only
+    multiply the rows it has to sum back together.
+    """
+
+    def record(self, *, category: BillableCategory, route: str, status_code: int) -> None: ...
+
+
 _MODEL_ID_HEADER: Final = b"x-litellm-model-id"
 
 # Ordered: a longer suffix that shares an ending with a shorter one must come
@@ -165,10 +187,11 @@ def _extract_model_id(headers: Sequence[tuple[bytes, bytes]]) -> str | None:
 
 class BillableRequestMetricsMiddleware:
     """
-    Pure ASGI middleware that records one billable request per 2xx response to a
-    billable endpoint. Modeled on InFlightRequestsMiddleware: it wraps `send`,
-    reads the final status and the x-litellm-model-id header off the
-    `http.response.start` message, and never blocks or fails the request path.
+    Pure ASGI middleware that classifies each request once and fans the result
+    out to the SGR sink (any status) and the billing recorder (2xx only).
+    Modeled on InFlightRequestsMiddleware: it wraps `send`, reads the final
+    status and the x-litellm-model-id header off the `http.response.start`
+    message, and never blocks or fails the request path.
     """
 
     def __init__(
@@ -176,6 +199,8 @@ class BillableRequestMetricsMiddleware:
         app: ASGIApp,
         recorder: BillingRecorder | None = None,
         recorder_factory: Callable[[], BillingRecorder | None] | None = None,
+        sink: GatewayRequestSink | None = None,
+        sink_factory: Callable[[], GatewayRequestSink | None] | None = None,
     ) -> None:
         self.app = app
         self.recorder = recorder
@@ -187,6 +212,12 @@ class BillableRequestMetricsMiddleware:
         self._recorder_factory = recorder_factory
         self._resolved = recorder_factory is None
         self._resolve_lock = threading.Lock()
+        # Resolved on the same schedule and for the same reason: the DB is not
+        # connected at import time, so the sink cannot be built there either.
+        self.sink = sink
+        self._sink_factory = sink_factory
+        self._sink_resolved = sink_factory is None
+        self._sink_resolve_lock = threading.Lock()
 
     def _resolve_recorder(self) -> BillingRecorder | None:
         if self._resolved:
@@ -200,13 +231,24 @@ class BillableRequestMetricsMiddleware:
                 self._resolved = True
         return self.recorder
 
+    def _resolve_sink(self) -> GatewayRequestSink | None:
+        if self._sink_resolved:
+            return self.sink
+        with self._sink_resolve_lock:
+            if not self._sink_resolved:
+                factory: Final = self._sink_factory
+                self.sink = factory() if factory is not None else self.sink
+                self._sink_resolved = True
+        return self.sink
+
     async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
         if scope["type"] != "http":
             await self.app(scope, receive, send)
             return
 
         recorder: Final = self._resolve_recorder()
-        if recorder is None:
+        sink: Final = self._resolve_sink()
+        if recorder is None and sink is None:
             await self.app(scope, receive, send)
             return
 
@@ -228,7 +270,13 @@ class BillableRequestMetricsMiddleware:
 
         await self.app(scope, receive, send_wrapper)
 
-        if 200 <= status_code < 300:
+        if sink is not None:
+            try:
+                sink.record(category=category, route=route, status_code=status_code)
+            except Exception:  # noqa: BLE001 -- metering must never fail a request that was already served
+                verbose_proxy_logger.warning("gateway request metering failed for %s", route, exc_info=True)
+
+        if recorder is not None and 200 <= status_code < 300:
             try:
                 recorder.record(category=category, route=route, status_code=status_code, model_id=model_id)
             except Exception:  # noqa: BLE001 -- metering must never fail a request that was already served

litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py

@@ -1,68 +1,124 @@
-from typing import Final
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Final
 
 import litellm
 from litellm._logging import verbose_router_logger
 from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (
     LiteLLM_ManagedVectorStore,
 )
+from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
 from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials
+from litellm.types.router import LiteLLMParamsTypedDict
+
+if TYPE_CHECKING:
+    from litellm.router import Router
+
+
+def _get_proxy_llm_router() -> "Router | None":
+    from litellm.proxy.proxy_server import llm_router
+
+    return llm_router
+
+
+def _get_str_value(values: dict[str, object] | None, key: str) -> str | None:
+    value: Final = values.get(key) if values is not None else None
+    return value if isinstance(value, str) else None
 
 
 class PassthroughEndpointRouter:
     """
-    Use this class to Set/Get credentials for pass-through endpoints
+    Use this class to Get credentials for pass-through endpoints
     """
 
-    def __init__(self):
-        self.credentials: dict[str, str] = {}
+    def __init__(
+        self,
+        llm_router_getter: "Callable[[], Router | None]" = _get_proxy_llm_router,
+    ):
+        self.llm_router_getter: Final = llm_router_getter
         self.deployment_key_to_vertex_credentials: dict[str, VertexPassThroughCredentials] = {}
         self.default_vertex_config: VertexPassThroughCredentials | None = None
 
-    def set_pass_through_credentials(
-        self,
-        custom_llm_provider: str,
-        api_base: str | None,
-        api_key: str | None,
-    ):
-        """
-        Set credentials for a pass-through endpoint. Used when a user adds a pass-through LLM endpoint on the UI.
-
-        Args:
-            custom_llm_provider: The provider of the pass-through endpoint
-            api_base: The base URL of the pass-through endpoint
-            api_key: The API key for the pass-through endpoint
-        """
-        credential_name: Final = self._get_credential_name_for_provider(
-            custom_llm_provider=custom_llm_provider,
-            region_name=self._get_region_name_from_api_base(api_base=api_base, custom_llm_provider=custom_llm_provider),
-        )
-        if api_key is None:
-            raise ValueError("api_key is required for setting pass-through credentials")
-        self.credentials[credential_name] = api_key
-
     def get_credentials(
         self,
         custom_llm_provider: str,
         region_name: str | None,
     ) -> str | None:
-        credential_name: Final = self._get_credential_name_for_provider(
+        deployment_api_key: Final = self._get_deployment_api_key(
             custom_llm_provider=custom_llm_provider,
             region_name=region_name,
         )
+        if deployment_api_key is not None:
+            return deployment_api_key
         verbose_router_logger.debug(
-            "Pass-through llm endpoints router, looking for credentials for %s", credential_name
+            "No pass-through deployment credentials found for %s, looking for env variable", custom_llm_provider
         )
-        if credential_name in self.credentials:
-            verbose_router_logger.debug("Found credentials for %s", credential_name)
-            return self.credentials[credential_name]
-        else:
-            verbose_router_logger.debug("No credentials found for %s, looking for env variable", credential_name)
-            _env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
-                custom_llm_provider=custom_llm_provider,
+        _env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
+            custom_llm_provider=custom_llm_provider,
+        )
+        return get_secret_str(_env_variable_name)
+
+    def _get_deployment_api_key(
+        self,
+        custom_llm_provider: str,
+        region_name: str | None,
+    ) -> str | None:
+        llm_router: Final = self.llm_router_getter()
+        if llm_router is None:
+            return None
+        deployments: Final = llm_router.get_model_list() or ()
+        return next(
+            (
+                api_key
+                for deployment in deployments
+                if (
+                    api_key := self._resolve_matching_deployment_api_key(
+                        litellm_params=deployment["litellm_params"],
+                        custom_llm_provider=custom_llm_provider,
+                        region_name=region_name,
+                    )
+                )
+                is not None
+            ),
+            None,
+        )
+
+    def _resolve_matching_deployment_api_key(
+        self,
+        litellm_params: LiteLLMParamsTypedDict,
+        custom_llm_provider: str,
+        region_name: str | None,
+    ) -> str | None:
+        if litellm_params.get("use_in_pass_through") is not True:
+            return None
+        if self._get_deployment_provider(litellm_params) != custom_llm_provider:
+            return None
+        credential_name: Final = litellm_params.get("litellm_credential_name")
+        credential_values: Final = (
+            CredentialAccessor.get_credential_values(credential_name) if credential_name is not None else None
+        )
+        api_base: Final = _get_str_value(credential_values, "api_base") or litellm_params.get("api_base")
+        deployment_region: Final = self._get_region_name_from_api_base(
+            custom_llm_provider=custom_llm_provider,
+            api_base=api_base,
+        )
+        if deployment_region != region_name:
+            return None
+        return _get_str_value(credential_values, "api_key") or litellm_params.get("api_key")
+
+    def _get_deployment_provider(self, litellm_params: LiteLLMParamsTypedDict) -> str | None:
+        model: Final = litellm_params.get("model")
+        if model is None:
+            return None
+        try:
+            _, provider, _, _ = litellm.get_llm_provider(
+                model=model,
+                custom_llm_provider=litellm_params.get("custom_llm_provider"),
             )
-            return get_secret_str(_env_variable_name)
+        except litellm.exceptions.BadRequestError:
+            return None
+        return provider
 
     def _get_vertex_env_vars(self) -> VertexPassThroughCredentials:
         """
@@ -165,15 +221,6 @@ class PassthroughEndpointRouter:
         else:
             return self.default_vertex_config
 
-    def _get_credential_name_for_provider(
-        self,
-        custom_llm_provider: str,
-        region_name: str | None,
-    ) -> str:
-        if region_name is None:
-            return f"{custom_llm_provider.upper()}_API_KEY"
-        return f"{custom_llm_provider.upper()}_{region_name.upper()}_API_KEY"
-
     def _get_region_name_from_api_base(
         self,
         custom_llm_provider: str,

litellm/proxy/pass_through_endpoints/passthrough_guardrails.py

@@ -7,7 +7,7 @@ Handles guardrail execution for passthrough endpoints with:
 - Automatic inheritance from org/team/key levels when enabled
 """
 
-from typing import Any, Final, Union
+from typing import Any, Final
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import (
@@ -19,10 +19,10 @@ from litellm.proxy.pass_through_endpoints.jsonpath_extractor import JsonPathExtr
 
 # Type for raw guardrails config input (before normalization)
 # Can be a list of names or a dict with settings
-PassThroughGuardrailsConfigInput = Union[
-    list[str],  # Simple list: ["guard-1", "guard-2"]
-    PassThroughGuardrailsConfig,  # Dict: {"guard-1": {"request_fields": [...]}}
-]
+PassThroughGuardrailsConfigInput = (
+    list[str]  # Simple list: ["guard-1", "guard-2"]
+    | PassThroughGuardrailsConfig  # Dict: {"guard-1": {"request_fields": [...]}}
+)
 
 
 class PassthroughGuardrailHandler:
@@ -246,7 +246,7 @@ class PassthroughGuardrailHandler:
         guardrails_to_run: Final[dict[str, bool]] = {}
 
         # Add passthrough-specific guardrails
-        for guardrail_name in normalized_config.keys():
+        for guardrail_name in normalized_config:
             guardrails_to_run[guardrail_name] = True
             verbose_proxy_logger.debug("Added passthrough-specific guardrail: %s", guardrail_name)
 

litellm/proxy/policy_engine/__init__.py

@@ -47,14 +47,12 @@ from litellm.proxy.policy_engine.policy_resolver import PolicyResolver
 from litellm.proxy.policy_engine.policy_validator import PolicyValidator
 
 __all__ = [
-    # Registries
-    "PolicyRegistry",
-    "get_policy_registry",
     "AttachmentRegistry",
-    "get_attachment_registry",
-    # Core components
+    "ConditionEvaluator",
     "PolicyMatcher",
+    "PolicyRegistry",
     "PolicyResolver",
     "PolicyValidator",
-    "ConditionEvaluator",
+    "get_attachment_registry",
+    "get_policy_registry",
 ]

litellm/proxy/prompts/prompt_registry.py

@@ -180,7 +180,7 @@ class InMemoryPromptRegistry:
         from litellm.proxy.prompts.prompt_endpoints import get_base_prompt_id
 
         prompts_to_delete: Final = [
-            pid for pid in self.IN_MEMORY_PROMPTS.keys() if get_base_prompt_id(prompt_id=pid) == base_prompt_id
+            pid for pid in self.IN_MEMORY_PROMPTS if get_base_prompt_id(prompt_id=pid) == base_prompt_id
         ]
 
         for pid in prompts_to_delete: