Merge remote-tracking branch 'origin/main' into litellm_pr48393_head

# Conflicts:
#	tests/unit/litellm_core_utils/test_streaming_handler.py

.agents/skills/database-access/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: database-access
 description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
+metadata:
+  internal: true
 ---
 
 # Database access

.github/workflows/interactions-spec-canary.yml

@@ -19,7 +19,7 @@ jobs:
     name: interactions-spec-canary
     if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
     permissions:
       contents: read
     outputs:
@@ -39,8 +39,11 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        uses: ./.github/actions/rust-bridge
+
       - name: Install dependencies
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
 
       - name: Run the compliance tests against Google's live spec
         id: compliance

.github/workflows/spend-logs-guard.yml

@@ -4,10 +4,6 @@ on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the P
   pull_request_target:
     branches:
       - main
-  pull_request_review:
-    types:
-      - submitted
-      - dismissed
 
 permissions:
   contents: read

.github/workflows/test-merge-smoke.yml

@@ -132,7 +132,7 @@ jobs:
   windows-sdk:
     name: windows-sdk
     runs-on: windows-2025
-    timeout-minutes: 10
+    timeout-minutes: 30
     env:
       LITELLM_LOCAL_MODEL_COST_MAP: "True"
     defaults:
@@ -154,8 +154,11 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        uses: ./.github/actions/rust-bridge
+
       - name: Install dependencies
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --group dev --python 3.11
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
 
       - name: Run the Windows tests
         run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests

.github/workflows/test-unit.yml

@@ -137,6 +137,7 @@ jobs:
         include:
           - shard: core-utils
             test-path: |-
+              tests/unit/accounting
               tests/unit/batch_completion
               tests/unit/decisions
               tests/unit/extraction
@@ -1306,6 +1307,8 @@ jobs:
           done
   openai-sdk-compat:
     name: OpenAI ${{ matrix.openai-version }} compatibility
+    needs: rust-bridge
+    if: ${{ !cancelled() }}
     permissions:
       contents: read
       pull-requests: read
@@ -1342,10 +1345,18 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/rust-bridge
+        with:
+          artifact: ${{ needs.rust-bridge.outputs.artifact }}
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group dev --group proxy-dev --extra proxy --no-install-workspace
+          cp rust-bridge-dist/litellm/rust_bridge/_native.* litellm/rust_bridge/
           uv pip install "openai==${OPENAI_VERSION:?}" --exclude-newer-package openai=2026-10-07T00:00:00Z
           uv run --no-sync python -c 'import os, openai; assert openai.__version__ == os.environ["OPENAI_VERSION"], openai.__version__'
 

ci_cd/spend_logs_guard.py

@@ -2,9 +2,12 @@
 needs an approving review from the spend-log owner before it can merge.
 
 CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
-so a path rule would put every migration on the owner. This gate reads the diff instead: when no
-added line in the watched files mentions the table it passes at once, otherwise it passes only once
-the owner's latest review on the pull request is an approval. The workflow re-runs on every review.
+so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma
+hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration
+hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes
+only once the owner's latest review on the pull request is an approval that still covers every spend-log line
+in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw).
+Reviews do not trigger the workflow, so the check is re-run by hand after approving.
 """
 
 from __future__ import annotations
@@ -13,8 +16,10 @@ import argparse
 import re
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
 from pathlib import Path
+from types import MappingProxyType
 from typing import Final
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter
@@ -25,7 +30,10 @@ WATCHED: Final = (
     "litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
 )
 MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
-TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs", re.IGNORECASE)
+TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
+DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
+MODEL_START: Final = re.compile(r"^model\s+LiteLLM_SpendLogs\s*\{")
+HUNK: Final = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@[^\n]*\n", re.MULTILINE)
 
 
 class Reviewer(BaseModel):
@@ -37,31 +45,77 @@ class Review(BaseModel):
     model_config = ConfigDict(frozen=True, extra="ignore")
     state: str
     user: Reviewer
+    commit_id: str | None = None
 
 
 _REVIEWS: Final = TypeAdapter(list[Review])
 
 
+@dataclass(frozen=True, slots=True)
+class Block:
+    """Lines of `model LiteLLM_SpendLogs { ... }` in one schema.prisma copy at the merge base and at the head."""
+
+    base: range
+    head: range
+
+
+@dataclass(frozen=True, slots=True)
+class Hunk:
+    removed: tuple[tuple[int, str], ...]
+    added: tuple[tuple[int, str], ...]
+
+
 def watched(path: str) -> bool:
     return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
 
 
-def _added_lines(section: str) -> tuple[str, ...]:
-    body: Final = section.split("\n@@", 1)
-    return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
+def model_block(schema: str) -> range:
+    """1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none."""
+    lines: Final = schema.splitlines()
+    starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line))
+    if not starts:
+        return range(0)
+    ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}"))
+    return range(starts[0], (ends[0] if ends else len(lines)) + 1)
 
 
-def spend_log_hits(diff: str) -> tuple[str, ...]:
-    """`path: added line` for every added line in a watched file that names the table."""
+def _hunks(section: str) -> tuple[Hunk, ...]:
+    headers: Final = tuple(HUNK.finditer(section))
+    ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),)
+    return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends))
+
+
+def _hunk(header: re.Match[str], body: str) -> Hunk:
+    lines: Final = body.splitlines()
+    return Hunk(
+        removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))),
+        added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))),
+    )
+
+
+def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    """`path: +line` or `path: -line` for every changed line that touches the table.
+
+    A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines,
+    merge base for removed ones), a migration line when it is added and names the table."""
     sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
-    return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
-
-
-def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
-    pairs: Final = tuple((_path_of(section), section) for section in sections)
+    paths: Final = tuple((_path_of(section), section) for section in sections)
     return tuple(
-        (path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
-    )  # comprehension-ok: path then its lines
+        hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks)
+    )  # comprehension-ok: path then its hits
+
+
+def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    if path not in blocks:
+        return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line))
+    block: Final = blocks[path]
+    removed: Final = tuple(
+        f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base
+    )  # comprehension-ok: hunk then its lines
+    added: Final = tuple(
+        f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head
+    )  # comprehension-ok: hunk then its lines
+    return removed + added
 
 
 def _path_of(section: str) -> str:
@@ -70,11 +124,33 @@ def _path_of(section: str) -> str:
 
 
 def approved_by(reviews: Sequence[Review], login: str) -> bool:
-    """True when `login`'s most recent review on the pull request is an approval."""
-    states: Final = tuple(review.state for review in reviews if review.user.login == login)
+    """True when `login`'s most recent review that changes GitHub's approval state is an approval.
+
+    COMMENTED and PENDING reviews leave an earlier approval in force on GitHub, so they are skipped here too.
+    """
+    own: Final = tuple(review.state for review in reviews if review.user.login == login)
+    states: Final = tuple(state for state in own if state in DECIDING_STATES)
     return bool(states) and states[-1] == "APPROVED"
 
 
+def latest_approval(reviews: Sequence[Review], login: str) -> Review | None:
+    """`login`'s most recent APPROVED review on the pull request, the one GitHub keeps in force after pushes."""
+    own: Final = tuple(review for review in reviews if review.user.login == login and review.state == "APPROVED")
+    return own[-1] if own else None
+
+
+def approval_covers(
+    approval: Review, head: str, hits: Sequence[str], hits_at: Callable[[str], Sequence[str] | None]
+) -> bool:
+    """True when the approval is on the head, or the head adds no spend-log line the approved commit did not."""
+    if approval.commit_id is None:
+        return False
+    if approval.commit_id == head:
+        return True
+    reviewed: Final = hits_at(approval.commit_id)
+    return reviewed is not None and set(hits) <= set(reviewed)
+
+
 def added_diff(base: str, head: str) -> str:
     result: Final = subprocess.run(
         ("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX),
@@ -85,6 +161,27 @@ def added_diff(base: str, head: str) -> str:
     return result.stdout
 
 
+def file_at(revision: str, path: str) -> str:
+    """Contents of `path` at `revision`, empty when the file does not exist there."""
+    result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True)
+    return result.stdout if result.returncode == 0 else ""
+
+
+def schema_blocks(base: str, head: str) -> Mapping[str, Block]:
+    return MappingProxyType(
+        {path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED}
+    )
+
+
+def fetched(commit: str) -> bool:
+    result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
+    return result.returncode == 0
+
+
+def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
+    return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None
+
+
 class Arguments(BaseModel):
     model_config = ConfigDict(frozen=True)
     base: str
@@ -100,18 +197,28 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
     parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
     args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
-    hits: Final = spend_log_hits(added_diff(args.base, args.head))
+    hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head))
     if not hits:
-        sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
+        sys.stdout.write(
+            "No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; "
+            "nothing to review.\n"
+        )
         return 0
-    if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver):
+    reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
+    approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None
+    if approval is not None and approval_covers(
+        approval, args.head, hits, lambda commit: reviewed_hits(args.base, commit)
+    ):
         sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n")
         return 0
     listed: Final = "".join(f"  {hit}\n" for hit in hits)
-    sys.stdout.write(
+    reason: Final = (
         f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n"
-        f"{listed}The check re-runs when a review is submitted.\n"
+        if approval is None
+        else f"LiteLLM_SpendLogs lines changed after @{args.approver} approved {approval.commit_id}, "
+        f"a new approving review is needed:\n"
     )
+    sys.stdout.write(f"{reason}{listed}After approving, re-run this job.\n")
     return 1
 
 

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -32,6 +32,7 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.llms.base_llm.guardrail_translation.utils import effective_skip_tool_message_for_guardrail
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails._content_utils import iter_tool_json_fields, map_tool_json_text, walk_user_text
 from litellm.types.guardrails import GuardrailEventHooks
@@ -753,7 +754,12 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         _masked_entity_count.set({})
 
         # Covers multimodal list content + Responses-API input.
-        inspected = walk_user_text(data, self.redact_text, self.redact_tool_payload) + self._redact_prompt(data)
+        inspected = walk_user_text(
+            data,
+            self.redact_text,
+            self.redact_tool_payload,
+            skip_tool_message=effective_skip_tool_message_for_guardrail(self),
+        ) + self._redact_prompt(data)
 
         if inspected == 0:
             # Image-only, empty-text, and unsupported payloads inspected

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -28,7 +28,7 @@ import httpx
 from fastapi import HTTPException
 from openai import APIConnectionError, AsyncOpenAI
 from openai.types.file_deleted import FileDeleted
-from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, Unpack
 
 import litellm
@@ -53,7 +53,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_metadata,
 )
 from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
-from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
 from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
@@ -309,6 +309,7 @@ def _is_xai_batch_output(
 class _ManagedFileRow(Protocol):
     unified_file_id: str
     file_object: OpenAIFileObject
+    model_mappings: Mapping[str, str]
     flat_model_file_ids: Sequence[str]
     storage_backend: Optional[str]
     storage_url: Optional[str]
@@ -362,6 +363,8 @@ class _ManagedResourceDatabase(Protocol):
     @property
     def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
 
+    async def query_raw(self, query: str, *args: object) -> object: ...
+
     @property
     def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
 
@@ -385,10 +388,83 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
+    return prisma_client.db
+
+
 def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedFileTableActions:
     return prisma_client.db.litellm_managedfiletable
 
 
+class _ListedFileRow(Protocol):
+    @property
+    def unified_file_id(self) -> str: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+    @property
+    def model_mappings(self) -> Mapping[str, object]: ...
+
+    @property
+    def storage_backend(self) -> str | None: ...
+
+
+class _ScopedFileRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    unified_file_id: str
+    file_object: JsonValue = None
+    model_mappings: Mapping[str, JsonValue]
+    storage_backend: str | None = None
+
+
+_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...])
+
+
+def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]:
+    match owner_filter:
+        case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}:
+            return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id)
+        case {"created_by": str() as user_id}:
+            return '"created_by" = $1', (user_id,)
+        case {"team_id": str() as team_id}:
+            return '"team_id" = $1', (team_id,)
+        case _:
+            return ("FALSE", ()) if owner_filter else ("TRUE", ())
+
+
+async def _scoped_file_rows(
+    database: _ManagedResourceDatabase,
+    owner_filter: Mapping[str, object],
+    model_mapping_keys: frozenset[str],
+    cursor_id: str | None,
+    direction: Literal["asc", "desc"],
+    take: int,
+) -> tuple[_ScopedFileRow, ...]:
+    owner_condition, owner_params = _owner_condition(owner_filter)
+    keys_param: Final = len(owner_params) + 1
+    comparison: Final = "<" if direction == "desc" else ">"
+    cursor_condition: Final = (
+        f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" '
+        f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) '
+        if cursor_id
+        else ""
+    )
+    order: Final = direction.upper()
+    rows: Final = await database.query_raw(
+        'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" '
+        f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) '
+        f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} '
+        f"LIMIT ${keys_param + (2 if cursor_id else 1)}",
+        *owner_params,
+        sorted(model_mapping_keys),
+        *((cursor_id,) if cursor_id else ()),
+        take,
+    )
+    return _SCOPED_FILE_ROWS.validate_python(rows)
+
+
 def _iter_provider_file_id_pairs(
     rows: Sequence[_ManagedFileRow],
     requested_provider_file_ids: frozenset[str],
@@ -2265,14 +2341,23 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         user_api_key_dict: UserAPIKeyAuth,
         limit: Optional[int] = None,
         after: Optional[str] = None,
+        model_scope: Optional[FileListModelScope] = None,
+        before: Optional[str] = None,
         **data: Dict,
     ) -> FileListPage:
         """List the managed files the caller owns, newest first.
 
         Pagination is keyset based on ``unified_file_id`` so a key that owns
         every file on the proxy still reads one bounded page at a time.
-        ``purpose`` is applied after parsing, because the managed file table
-        keeps it inside the ``file_object`` blob instead of a column, and rows
+        ``after`` pages toward older files and ``before`` toward newer ones,
+        each page still ordered newest first.
+        ``model_scope`` is applied in the query, so only rows whose
+        ``model_mappings`` name one of its deployment ids or model names, and
+        storage-backed rows, leave the database; a storage-backed row then
+        stays only when its model name is one the router sends to the listed
+        provider. ``purpose`` is applied after parsing, because the managed
+        file table keeps it inside the ``file_object`` blob instead of a
+        column. Rows
         whose blob will not parse drop out there too, so a chunk of rows can
         yield fewer matches than the page holds. Successive chunks are read
         until the page is full or the caller's rows run out, which keeps
@@ -2281,9 +2366,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         scan has to continue past it, the chunk widens to
         ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
         that many rows instead of one per page. That bound is per query, not
-        per request: the work is still linear in the rows the caller owns, and
-        a filter matching nothing reads every one of them, with no index
-        covering either the owner filter or the sort.
+        per request: a ``purpose`` filter matching nothing still reads every
+        row that passes the query.
         """
         validate_file_list_limit(limit)
         validate_file_list_purpose(purpose)
@@ -2292,36 +2376,60 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if owner_filter is None:
             return FileListPage.model_validate(build_list_page([]))
 
-        if after:
-            cursor_row = await _managed_file_table(self.prisma_client).find_first(
-                where={**owner_filter, "unified_file_id": after}
+        if after and before:
+            raise ProxyException(
+                message="Pass either 'after' (or 'after_id') or 'before_id' when listing files, not both.",
+                type="invalid_request_error",
+                param="before_id",
+                code=400,
+                openai_code="invalid_value",
+            )
+        cursor_param: Final = "before_id" if before else "after"
+        cursor: Final = before or after
+        database: Final = _managed_database(self.prisma_client)
+        if cursor:
+            cursor_row = await database.litellm_managedfiletable.find_first(
+                where={**owner_filter, "unified_file_id": cursor}
             )
             if cursor_row is None:
                 raise ProxyException(
-                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    message=f"Invalid '{cursor_param}' cursor: no file found with id '{cursor}'.",
                     type="invalid_request_error",
-                    param="after",
+                    param=cursor_param,
                     code=400,
                     openai_code="invalid_value",
                 )
 
+        if model_scope is not None and not model_scope.model_mapping_keys:
+            return FileListPage.model_validate(build_list_page([]))
+
         page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
         matches: Final[List[OpenAIFileObject]] = []
-        cursor_id = after
+        cursor_id = cursor
         chunk_size = page_size + 1
+        direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc"
 
         while len(matches) <= page_size:
             cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
-            chunk = await _managed_file_table(self.prisma_client).find_many(
-                where=owner_filter,
-                take=chunk_size,
-                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
-                **cursor_args,
+            chunk: Sequence[_ListedFileRow] = (
+                await _scoped_file_rows(
+                    database, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size
+                )
+                if model_scope is not None
+                else await database.litellm_managedfiletable.find_many(
+                    where=owner_filter,
+                    take=chunk_size,
+                    order=[{"created_at": direction}, {"unified_file_id": direction}],
+                    **cursor_args,
+                )
             )
             matches.extend(
                 _public_file_object(parsed_file_object, row.unified_file_id)
                 for row in chunk
-                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                if (
+                    model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
+                )
+                and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
                 and (purpose is None or parsed_file_object.purpose == purpose)
             )
             if len(chunk) < chunk_size:
@@ -2329,7 +2437,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             cursor_id = chunk[-1].unified_file_id
             chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
 
-        return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
+        page: Final = matches[:page_size]
+        return FileListPage.model_validate(
+            build_list_page(page[::-1] if before else page, has_more=len(matches) > page_size)
+        )
 
     def _is_batch_polling_enabled(self) -> bool:
         """

gateway/Dockerfile

@@ -153,7 +153,7 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
     python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
-    python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
+    python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__"
 
 USER nonroot
 

litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py

@@ -343,11 +343,16 @@ def btree_max_entry_bytes(block_size: int) -> int:
 
 
 def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed":
+    """The bytes a value takes in the index entry: its length, or its stored size when
+    Postgres compressed it, since `index_form_tuple` keeps the compressed bytes."""
     from psycopg import sql
 
-    if column.type_name in _OCTET_LENGTH_TYPES:
-        return sql.SQL("octet_length({})").format(identifier)
-    return sql.SQL("octet_length({}::text)").format(identifier)
+    length: Final = (
+        sql.SQL("octet_length({})").format(identifier)
+        if column.type_name in _OCTET_LENGTH_TYPES
+        else sql.SQL("octet_length({}::text)").format(identifier)
+    )
+    return sql.SQL("least({}, pg_column_size({}))").format(length, identifier)
 
 
 def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed":

litellm-proxy-extras/litellm_proxy_extras/migrations/20261021000000_managed_file_created_by_created_at_index/migration.sql

@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_ManagedFileTable_created_by_created_at_idx"
+ON "LiteLLM_ManagedFileTable" ("created_by", "created_at" DESC);

litellm-proxy-extras/litellm_proxy_extras/migrations/20261027000000_add_accounting_ledger_tables/migration.sql

@@ -0,0 +1,47 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE TYPE "AccountingEventKind" AS ENUM ('inbound_usage', 'outbound_usage', 'cost');
+
+CREATE TYPE "AccountingDeliveryState" AS ENUM ('pending', 'applied', 'parked');
+
+CREATE TYPE "AccountingDeliveryTarget" AS ENUM ('LiteLLM_VerificationToken', 'LiteLLM_UserTable', 'LiteLLM_TeamTable', 'LiteLLM_TeamMembership', 'LiteLLM_OrganizationTable', 'LiteLLM_OrganizationMembership', 'LiteLLM_EndUserTable', 'LiteLLM_AgentsTable', 'LiteLLM_TagTable', 'LiteLLM_ProjectTable', 'LiteLLM_ModelAccessGroupBudgetTable', 'LiteLLM_BudgetTable', 'LiteLLM_BudgetWindowSpend', 'LiteLLM_DailyUserSpend', 'LiteLLM_DailyTeamSpend', 'LiteLLM_DailyOrganizationSpend', 'LiteLLM_DailyEndUserSpend', 'LiteLLM_DailyAgentSpend', 'LiteLLM_DailyTagSpend', 'LiteLLM_DailyModelUsage', 'LiteLLM_DailyToolSpend', 'LiteLLM_AutoRouterDailySpend');
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingEvent" (
+    "event_id" TEXT NOT NULL,
+    "kind" "AccountingEventKind" NOT NULL,
+    "source_event_id" TEXT,
+    "schema_version" INTEGER NOT NULL,
+    "payload_hash" TEXT NOT NULL,
+    "occurred_at" TIMESTAMP(3) NOT NULL,
+    "recorded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "payload" JSONB NOT NULL,
+
+    CONSTRAINT "LiteLLM_AccountingEvent_schema_version_check" CHECK ("schema_version" > 0),
+    CONSTRAINT "LiteLLM_AccountingEvent_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingDelivery" (
+    "event_id" TEXT NOT NULL,
+    "target" "AccountingDeliveryTarget" NOT NULL,
+    "version" INTEGER NOT NULL,
+    "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "applied_at" TIMESTAMP(3),
+    "last_error" TEXT,
+
+    CONSTRAINT "LiteLLM_AccountingDelivery_version_check" CHECK ("version" > 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_attempts_check" CHECK ("attempts" >= 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_pkey" PRIMARY KEY ("event_id","target","version")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_recorded_at_idx" ON "LiteLLM_AccountingEvent"("recorded_at");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_source_event_id_idx" ON "LiteLLM_AccountingEvent"("kind", "source_event_id");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingDelivery_target_state_next_attempt_at_idx" ON "LiteLLM_AccountingDelivery"("target", "state", "next_attempt_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
   @@index([unified_file_id])
   @@index([flat_model_file_ids], type: Gin)
   @@index([team_id, created_at(sort: Desc)])
+  @@index([created_by, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
   @@index([window_end])
   @@index([created_at])
 }
+
+enum AccountingEventKind {
+  inbound_usage
+  outbound_usage
+  cost
+}
+
+enum AccountingDeliveryState {
+  pending
+  applied
+  parked
+}
+
+enum AccountingDeliveryTarget {
+  LiteLLM_VerificationToken
+  LiteLLM_UserTable
+  LiteLLM_TeamTable
+  LiteLLM_TeamMembership
+  LiteLLM_OrganizationTable
+  LiteLLM_OrganizationMembership
+  LiteLLM_EndUserTable
+  LiteLLM_AgentsTable
+  LiteLLM_TagTable
+  LiteLLM_ProjectTable
+  LiteLLM_ModelAccessGroupBudgetTable
+  LiteLLM_BudgetTable
+  LiteLLM_BudgetWindowSpend
+  LiteLLM_DailyUserSpend
+  LiteLLM_DailyTeamSpend
+  LiteLLM_DailyOrganizationSpend
+  LiteLLM_DailyEndUserSpend
+  LiteLLM_DailyAgentSpend
+  LiteLLM_DailyTagSpend
+  LiteLLM_DailyModelUsage
+  LiteLLM_DailyToolSpend
+  LiteLLM_AutoRouterDailySpend
+}
+
+// Durable accounting ledger. One row per usage or cost event; payload holds the
+// private event envelope versioned by schema_version. Usage rows retain request-scoped
+// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
+model LiteLLM_AccountingEvent {
+  event_id        String              @id
+  kind            AccountingEventKind
+  source_event_id String?
+  schema_version  Int
+  payload_hash    String
+  occurred_at     DateTime
+  recorded_at     DateTime            @default(now())
+  payload         Json
+
+  @@index([recorded_at])
+  @@index([kind, source_event_id])
+}
+
+// One row per (event, target table, version): pending work for the projector and,
+// once applied in the same transaction as the target write, the receipt that stops a retry.
+model LiteLLM_AccountingDelivery {
+  event_id        String
+  target          AccountingDeliveryTarget
+  version         Int
+  state           AccountingDeliveryState  @default(pending)
+  attempts        Int                      @default(0)
+  next_attempt_at DateTime                 @default(now())
+  applied_at      DateTime?
+  last_error      String?
+
+  @@id([event_id, target, version])
+  @@index([target, state, next_attempt_at])
+}

litellm-rust/.agents/skills/rust-error/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-error
 description: Define, wrap, convert, classify, report or test Rust error types in litellm-rust with thiserror. Use when adding an error type or variant, mapping one crate's error into another, turning an error into a gateway response or Python exception, deciding retry or fault from an error, or replacing call sites that build errors from strings
+metadata:
+  internal: true
 ---
 
 # Rust errors

litellm-rust/.agents/skills/rust-gateway/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-gateway
 description: Add or change an HTTP route, extractor, response envelope, stream or router mount in the litellm-rust gateway crates (crates/gateway and crates/gateway-*). Use when adding an endpoint, parsing a request body, choosing an error envelope for an API surface, relaying SSE, or writing a router-level test
+metadata:
+  internal: true
 ---
 
 # Rust gateway

litellm-rust/.agents/skills/rust-string-enums/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-string-enums
 description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas
+metadata:
+  internal: true
 ---
 
 # Rust string enums

litellm-rust/.agents/skills/rust-tracing/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-tracing
 description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery
+metadata:
+  internal: true
 ---
 
 # Rust tracing

litellm-rust/crates/core-utils/src/core_helpers.rs

@@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
     ("max_tokens", "length"),
     ("refusal", "content_filter"),
     ("compaction", "length"),
+    ("model_context_window_exceeded", "length"),
     ("guardrail_intervened", "content_filter"),
     ("content_filtered", "content_filter"),
     ("content_filter", "content_filter"),
@@ -74,6 +75,7 @@ mod tests {
     #[case::end_turn("end_turn", "stop")]
     #[case::stop_sequence("stop_sequence", "stop")]
     #[case::max_tokens("max_tokens", "length")]
+    #[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
     #[case::refusal("refusal", "content_filter")]
     #[case::guardrail_intervened("guardrail_intervened", "content_filter")]
     #[case::content_filtered("content_filtered", "content_filter")]

litellm-rust/crates/python-bridge/src/logger/tests.rs

@@ -187,6 +187,7 @@ fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication
         locals
             .set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
             .unwrap();
+        locals.set_item("native", crate::native_module(py)).unwrap();
         let importable = py
             .eval(
                 c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
@@ -205,6 +206,7 @@ import asyncio
 import logging
 import sys
 sys.path.insert(0, repo_root)
+sys.modules['litellm.rust_bridge._native'] = native
 import litellm
 from litellm._logging import verbose_logger, session_id_var, trace_id_var
 

litellm/accounting/__init__.py

@@ -0,0 +1,62 @@
+"""Public accounting contracts. Importing these values does not enable accounting."""
+
+from litellm.accounting.measurements import measurements_for
+from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter
+from litellm.accounting.types import (
+    AdmissionDecision,
+    AggregateQuery,
+    AggregateRow,
+    CostEvent,
+    DeliveryState,
+    DeliveryTarget,
+    Dimension,
+    DimensionFilter,
+    DimensionName,
+    EventKind,
+    Granularity,
+    InboundUsageEvent,
+    Measurement,
+    Metric,
+    MetricName,
+    Multiplier,
+    NamedCost,
+    OutboundUsageEvent,
+    Outcome,
+    PricedQuantity,
+    PricingSnapshot,
+    Rate,
+    Unpriced,
+    validate_aggregate_rows,
+)
+
+__all__ = [
+    "AdmissionDecision",
+    "AggregateQuery",
+    "AggregateRow",
+    "BudgetEnforcer",
+    "CostAggregator",
+    "CostCompiler",
+    "CostEvent",
+    "DeliveryState",
+    "DeliveryTarget",
+    "Dimension",
+    "DimensionFilter",
+    "DimensionName",
+    "EventKind",
+    "Granularity",
+    "InboundUsageEvent",
+    "Measurement",
+    "Metric",
+    "MetricName",
+    "Multiplier",
+    "NamedCost",
+    "OutboundUsageEvent",
+    "Outcome",
+    "PricedQuantity",
+    "PricingSnapshot",
+    "Rate",
+    "Unpriced",
+    "UsageMeter",
+    "measurements_for",
+    "validate_aggregate_rows",
+]

litellm/accounting/measurements.py

@@ -0,0 +1,13 @@
+from typing import Final
+
+from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent
+
+
+def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]:
+    """Return the complete contribution set for one event, ready for one apply call."""
+    metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics
+    occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at
+    return tuple(
+        Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric)
+        for metric in metrics or ()
+    )

litellm/accounting/protocols.py

@@ -0,0 +1,85 @@
+"""Contracts only. Implementations own persistence, pricing, projections and live budgets.
+
+The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
+Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
+model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
+These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
+Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
+T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
+"""
+
+from typing import Protocol
+
+from litellm.accounting.types import (
+    AdmissionDecision,
+    AggregateQuery,
+    AggregateRow,
+    CostEvent,
+    Dimension,
+    InboundUsageEvent,
+    Measurement,
+    Metric,
+    OutboundUsageEvent,
+    Unpriced,
+)
+
+
+class UsageMeter(Protocol):
+    async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
+        """Acknowledge commit, no-op identical IDs and reject conflicting content.
+
+        Persist the event and its request-scoped pricing context before acknowledging.
+        HTTP acknowledgement is a separate policy. Unknown usage remains None.
+        """
+        ...
+
+
+class CostCompiler(Protocol):
+    async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
+        """Reuse the committed charge or finalize one charge and its delivery intent.
+
+        Resolve captured request-scoped context by event identity and cost-map rates
+        at compilation. Persist the resolved inputs in the required snapshot. Replay
+        reuses the committed result and never consumes free allowance again.
+        Missing pricing is Unpriced, never an inferred zero.
+        """
+        ...
+
+
+class CostAggregator(Protocol):
+    async def apply(self, measurements: tuple[Measurement, ...]) -> None:
+        """Apply one source event's complete contribution set per target/version.
+
+        Each target commits all contributions and its receipt atomically. A receipt
+        for the first metric must not suppress the rest. Usage need not wait for cost.
+        """
+        ...
+
+    async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
+        """Return disjoint [start, end) rows per group using existing daily projections.
+
+        Filters combine with AND, values within a filter with IN. DAY emits daily
+        buckets; None emits whole-interval totals. Reject unsupported metric/scope
+        combinations. Different groups may overlap, including non-additive tags.
+        """
+        ...
+
+
+class BudgetEnforcer(Protocol):
+    async def reserve(
+        self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...]
+    ) -> AdmissionDecision:
+        """Apply existing admission policies and estimates; duplicate IDs add no hold."""
+        ...
+
+    async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None:
+        """Settle the committed charge with atomic deduplication and hold mutation.
+
+        Preserve scope/window attribution and retry unfinished settlement after
+        restart. SQL projection receipts do not provide live-counter atomicity.
+        """
+        ...
+
+    async def release(self, reservation_id: str) -> None:
+        """Idempotently release unused exposure at the request boundary, preserving charges."""
+        ...

litellm/accounting/types.py

@@ -0,0 +1,428 @@
+"""Accounting values use UTC instants, decimal strings on the wire and USD charges."""
+
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from decimal import Decimal
+from enum import Enum
+from typing import Annotated, Final, Literal, TypeAlias
+
+from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator
+from typing_extensions import Self
+
+from litellm.types.utils import CallTypes
+
+_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True)
+Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")]
+Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)]
+
+
+def _utc(value: datetime) -> datetime:
+    return value.astimezone(timezone.utc)
+
+
+Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)]
+
+
+class MetricName(str, Enum):
+    """Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens."""
+
+    REQUESTS = "requests"
+    SUCCESSFUL_REQUESTS = "successful_requests"
+    FAILED_REQUESTS = "failed_requests"
+    PROMPT_TOKENS = "prompt_tokens"
+    COMPLETION_TOKENS = "completion_tokens"
+    REASONING_TOKENS = "reasoning_tokens"
+    TOTAL_TOKENS = "total_tokens"
+    CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens"
+    CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens"
+    CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr"
+    SPEND = "spend"
+
+
+class DimensionName(str, Enum):
+    KEY = "key"
+    USER = "user"
+    TEAM = "team"
+    ORGANIZATION = "organization"
+    END_USER = "end_user"
+    MODEL = "model"
+    MODEL_GROUP = "model_group"
+    DEPLOYMENT = "deployment"
+    PROVIDER = "provider"
+    TOOL = "tool"
+    AGENT = "agent"
+    PROJECT = "project"
+    MODEL_ACCESS_GROUP = "model_access_group"
+    API_BASE = "api_base"
+    SERVICE_TIER = "service_tier"
+    TAG = "tag"
+
+
+class Outcome(str, Enum):
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+class Granularity(str, Enum):
+    DAY = "day"
+
+
+class EventKind(str, Enum):
+    INBOUND_USAGE = "inbound_usage"
+    OUTBOUND_USAGE = "outbound_usage"
+    COST = "cost"
+
+
+class DeliveryState(str, Enum):
+    PENDING = "pending"
+    APPLIED = "applied"
+    PARKED = "parked"
+
+
+class DeliveryTarget(str, Enum):
+    """One member per existing table the projector writes; each is applied and receipted independently."""
+
+    KEY_SPEND = "LiteLLM_VerificationToken"
+    USER_SPEND = "LiteLLM_UserTable"
+    TEAM_SPEND = "LiteLLM_TeamTable"
+    TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership"
+    ORGANIZATION_SPEND = "LiteLLM_OrganizationTable"
+    ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership"
+    END_USER_SPEND = "LiteLLM_EndUserTable"
+    AGENT_SPEND = "LiteLLM_AgentsTable"
+    TAG_SPEND = "LiteLLM_TagTable"
+    PROJECT_SPEND = "LiteLLM_ProjectTable"
+    MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable"
+    BUDGET_SPEND = "LiteLLM_BudgetTable"
+    BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend"
+    DAILY_USER_SPEND = "LiteLLM_DailyUserSpend"
+    DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend"
+    DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend"
+    DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend"
+    DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend"
+    DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend"
+    DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage"
+    DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend"
+    AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend"
+
+
+_SUBSET_PAIRS: Final = (
+    (MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS),
+    (MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS),
+)
+
+
+def _check_subsets(values: Mapping[str, Decimal]) -> None:
+    for part, whole in _SUBSET_PAIRS:
+        if part.value in values and whole.value in values and values[part.value] > values[whole.value]:
+            raise ValueError(f"{part.value} cannot exceed {whole.value}")
+
+
+class Metric(BaseModel):
+    model_config = _CONFIG
+
+    name: MetricName
+    value: Quantity
+    unit: Literal["count", "USD"] = "count"
+
+    @model_validator(mode="after")
+    def validate_unit(self) -> Self:
+        if (self.name is MetricName.SPEND) != (self.unit == "USD"):
+            raise ValueError("SPEND requires USD; usage metrics require count")
+        if self.unit == "count" and self.value != self.value.to_integral_value():
+            raise ValueError("Count metrics require integral values")
+        return self
+
+
+class Dimension(BaseModel):
+    model_config = _CONFIG
+
+    name: DimensionName
+    value: Identifier
+
+
+_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP})
+
+
+def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]:
+    if len(frozenset(values)) != len(values):
+        raise ValueError("Duplicate dimension values")
+    single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS)
+    if len(frozenset(single_names)) != len(single_names):
+        raise ValueError("Only tag and model access group dimensions may have multiple values")
+    return values
+
+
+Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)]
+_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS))
+
+
+def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]:
+    if len(frozenset(value.name for value in values)) != len(values):
+        raise ValueError("Duplicate metric names")
+    return values
+
+
+Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)]
+
+
+def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None:
+    if end < start or (end == start and not allow_empty):
+        raise ValueError("End must follow start")
+
+
+class InboundUsageEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    request_id: Identifier
+    call_type: CallTypes
+    started_at: Instant
+    ended_at: Instant
+    outcome: Outcome
+    dimensions: Dimensions
+    metrics: Metrics
+
+    @model_validator(mode="after")
+    def validate_caller_activity(self) -> Self:
+        _interval(self.started_at, self.ended_at, allow_empty=True)
+        if any(metric.name not in _CALLER_METRICS for metric in self.metrics):
+            raise ValueError("Inbound events own only caller request metrics")
+        if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics):
+            raise ValueError("Inbound events count exactly one caller request")
+        for metric in self.metrics:
+            if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED):
+                raise ValueError("Successful request count must match outcome")
+            if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED):
+                raise ValueError("Failed request count must match outcome")
+        return self
+
+
+class OutboundUsageEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    attempt_id: Identifier
+    request_id: Identifier | None
+    retry_of_attempt_id: Identifier | None
+    call_type: CallTypes
+    started_at: Instant
+    ended_at: Instant
+    outcome: Outcome
+    dimensions: Dimensions
+    metrics: Metrics | None
+
+    @model_validator(mode="after")
+    def validate_attempt_usage(self) -> Self:
+        _interval(self.started_at, self.ended_at, allow_empty=True)
+        if self.retry_of_attempt_id == self.attempt_id:
+            raise ValueError("An attempt cannot retry itself")
+        if self.metrics is not None and any(
+            metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics
+        ):
+            raise ValueError("Outbound events own only attempt usage metrics")
+        if self.metrics is not None:
+            _check_subsets({metric.name.value: metric.value for metric in self.metrics})
+        return self
+
+
+PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"]
+
+
+class Rate(BaseModel):
+    model_config = _CONFIG
+
+    measure: Identifier
+    unit: PricingUnit
+    usd_per_unit: Quantity
+
+
+class PricedQuantity(BaseModel):
+    """A pricing input in its billing unit. Only seconds may be fractional."""
+
+    model_config = _CONFIG
+
+    measure: Identifier
+    unit: PricingUnit
+    value: Quantity
+
+    @model_validator(mode="after")
+    def validate_discrete_quantity(self) -> Self:
+        if self.unit != "second" and self.value != self.value.to_integral_value():
+            raise ValueError("Discrete priced quantities require integral values")
+        return self
+
+
+class Multiplier(BaseModel):
+    """A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5."""
+
+    model_config = _CONFIG
+
+    name: Identifier
+    value: Identifier
+    factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)]
+
+
+class NamedCost(BaseModel):
+    """A free-form extra charge by name, e.g. azure_model_router_flat_cost."""
+
+    model_config = _CONFIG
+
+    name: Identifier
+    amount: Quantity
+
+
+class PricingSnapshot(BaseModel):
+    """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
+
+    Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
+    resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
+    Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
+    Component sums and float tolerances belong to reconciliation, not this structural contract.
+    """
+
+    model_config = _CONFIG
+
+    schema_version: Literal[1] = 1
+    calculator_version: Identifier
+    model: Identifier
+    provider: Identifier | None
+    pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"]
+    priced_model_key: Identifier
+    multipliers: tuple[Multiplier, ...] = ()
+    rates: tuple[Rate, ...] = ()
+    usage: tuple[PricedQuantity, ...] = ()
+    input_cost: Quantity | None = None
+    output_cost: Quantity | None = None
+    cache_read_cost: Quantity | None = None
+    cache_creation_cost: Quantity | None = None
+    reasoning_cost: Quantity | None = None
+    tool_usage_cost: Quantity | None = None
+    guardrail_cost: Quantity | None = None
+    additional_costs: tuple[NamedCost, ...] = ()
+    original_cost: Quantity | None = None
+    discount_percent: Quantity | None = None
+    discount_amount: Quantity | None = None
+    margin_percent: Quantity | None = None
+    margin_fixed_amount: Quantity | None = None
+    margin_total_amount: Quantity | None = None
+    upstream_reported_cost: Quantity | None = None
+    total_cost: Quantity
+
+    @model_validator(mode="after")
+    def validate_snapshot(self) -> Self:
+        rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates)
+        quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage)
+        if len(rate_keys) != len(self.rates):
+            raise ValueError("Duplicate rate measures and units")
+        if len(quantity_keys) != len(self.usage):
+            raise ValueError("Duplicate priced quantity measures and units")
+        if not rate_keys.issubset(quantity_keys):
+            raise ValueError("Every rate requires one priced quantity with the same measure and unit")
+        if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs):
+            raise ValueError("Duplicate additional cost names")
+        if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers):
+            raise ValueError("Duplicate multiplier names")
+        _check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"})
+        return self
+
+
+class CostEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    usage_event_id: Identifier
+    occurred_at: Instant
+    dimensions: Dimensions
+    charges: Metrics
+    snapshot: PricingSnapshot
+
+    @model_validator(mode="after")
+    def validate_charge(self) -> Self:
+        if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND:
+            raise ValueError("A cost event requires one authoritative USD spend metric")
+        if self.event_id == self.usage_event_id:
+            raise ValueError("Cost and usage events require distinct identities")
+        if self.snapshot.total_cost != self.charges[0].value:
+            raise ValueError("Snapshot total must equal the authoritative spend")
+        return self
+
+
+class Unpriced(BaseModel):
+    model_config = _CONFIG
+
+    usage_event_id: Identifier
+    reason: Identifier
+
+
+class Measurement(BaseModel):
+    model_config = _CONFIG
+
+    source_event_id: Identifier
+    occurred_at: Instant
+    dimensions: Dimensions
+    metric: Metric
+
+
+class DimensionFilter(BaseModel):
+    model_config = _CONFIG
+
+    name: DimensionName
+    values: Annotated[tuple[Identifier, ...], Field(min_length=1)]
+
+
+class AggregateQuery(BaseModel):
+    model_config = _CONFIG
+
+    metric: MetricName
+    filters: tuple[DimensionFilter, ...]
+    group_by: tuple[DimensionName, ...]
+    start: Instant
+    end: Instant
+    granularity: Granularity | None
+
+    @model_validator(mode="after")
+    def validate_query(self) -> Self:
+        _interval(self.start, self.end)
+        if len(frozenset(item.name for item in self.filters)) != len(self.filters):
+            raise ValueError("Use one IN filter per dimension; filters combine with AND")
+        if len(frozenset(self.group_by)) != len(self.group_by):
+            raise ValueError("Duplicate group-by dimensions")
+        if any(
+            (instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0)
+            for instant in (self.start, self.end)
+        ):
+            raise ValueError("Initial reporting queries require UTC day boundaries")
+        return self
+
+
+class AggregateRow(BaseModel):
+    model_config = _CONFIG
+
+    start: Instant
+    end: Instant
+    dimensions: Dimensions
+    value: Quantity
+
+    @model_validator(mode="after")
+    def validate_bounds(self) -> Self:
+        _interval(self.start, self.end)
+        return self
+
+
+def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]:
+    """Validate [start, end) buckets per group; overlapping tag groups are not additive."""
+    groups: Final = frozenset(frozenset(row.dimensions) for row in rows)
+    for group in groups:
+        ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start)
+        if any(left.end > right.start for left, right in zip(ordered, ordered[1:])):
+            raise ValueError("Aggregate buckets overlap within a dimension group")
+    return rows
+
+
+class AdmissionDecision(BaseModel):
+    model_config = _CONFIG
+
+    allowed: bool
+    reason: Identifier | None = None

litellm/caching/in_memory_cache.py

@@ -274,7 +274,9 @@ class InMemoryCache(BaseCache):
     ) -> list[float] | None:
         results: Final = []
         for increment in increment_list:
-            result = await self.async_increment(increment["key"], increment["increment_value"], **kwargs)
+            result = await self.async_increment(
+                increment["key"], increment["increment_value"], ttl=increment["ttl"], **kwargs
+            )
             results.append(result)
         return results
 

litellm/constants.py

@@ -129,6 +129,7 @@ MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
 S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
 S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
 MAX_FILE_LIST_LIMIT: Final = 10000
+ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
 GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10

litellm/images/main.py

@@ -1,9 +1,9 @@
 import asyncio
 import contextvars
 import importlib
-from collections.abc import Coroutine, Mapping
+from collections.abc import Collection, Coroutine, Mapping
 from functools import partial
-from typing import TYPE_CHECKING, Final, Literal, cast, overload
+from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
 
 from pydantic import TypeAdapter, ValidationError
 
@@ -61,6 +61,7 @@ from litellm.types.utils import (
     ProviderRoutingHints,
 )
 from litellm.utils import (
+    EXTRA_BODY_ROUTING_KEYS,
     ImageResponse,
     ProviderConfigManager,
     get_llm_provider,
@@ -466,7 +467,7 @@ def image_generation(
             litellm.LlmProviders.QWENCLOUD,
             litellm.LlmProviders.QWEN_AI_PLATFORM,
             litellm.LlmProviders.EDENAI,
-        ):
+        ) or (custom_llm_provider == litellm.LlmProviders.TENCENT and image_generation_config is not None):
             if image_generation_config is None:
                 raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
 
@@ -807,6 +808,32 @@ def image_variation(
     return response
 
 
+_IMAGE_EDIT_OPENAI_PARAMS: Final = frozenset(get_type_hints(ImageEditOptionalRequestParams))
+_DROP_PARAM_KWARGS: Final = frozenset({"drop_params", "additional_drop_params"})
+_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter[tuple[str, ...]](tuple[str, ...])
+
+
+def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]:
+    try:
+        return _ADDITIONAL_DROP_PARAMS.validate_python(additional_drop_params)
+    except ValidationError:
+        return ()
+
+
+def _tokenhub_passthrough_edit_params(
+    non_default_params: Mapping[str, object],
+    extra_body: Mapping[str, object] | None,
+    mapped_params: Collection[str],
+    additional_drop_params: Collection[str] | None,
+) -> Mapping[str, object]:
+    dropped: Final = frozenset(additional_drop_params or ())
+    unmapped_excluded: Final = _DROP_PARAM_KWARGS | _IMAGE_EDIT_OPENAI_PARAMS | frozenset(mapped_params) | dropped
+    return {
+        **{key: value for key, value in non_default_params.items() if key not in unmapped_excluded},
+        **{key: value for key, value in (extra_body or {}).items() if key not in dropped | EXTRA_BODY_ROUTING_KEYS},
+    }
+
+
 @client
 def image_edit(
     image: FileTypes | list[FileTypes] | None = None,
@@ -935,22 +962,24 @@ def image_edit(
             raise ValueError(f"image edit is not supported for {custom_llm_provider}")
 
         local_vars.update(kwargs)
+        provider_supported_params: Final[frozenset[str]] = frozenset(
+            image_edit_provider_config.get_supported_openai_params(model)
+        )
         # Get ImageEditOptionalRequestParams with only valid parameters
         image_edit_optional_params: Final[ImageEditOptionalRequestParams] = (
             _get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
                 local_vars,
-                provider_supported_params=frozenset(
-                    image_edit_provider_config.get_supported_openai_params(model)
-                ).intersection(non_default_params),
+                provider_supported_params=provider_supported_params.intersection(non_default_params),
             )
         )
         # Get optional parameters for the responses API
+        additional_drop_params: Final = _listed_drop_params(kwargs.get("additional_drop_params"))
         image_edit_request_params: Final[dict] = _get_ImageEditRequestUtils().get_optional_params_image_edit(
             model=model,
             image_edit_provider_config=image_edit_provider_config,
             image_edit_optional_params=image_edit_optional_params,
             drop_params=kwargs.get("drop_params"),
-            additional_drop_params=kwargs.get("additional_drop_params"),
+            additional_drop_params=additional_drop_params,
         )
 
         if image_edit_provider_config.use_multipart_form_data() and (
@@ -1039,7 +1068,19 @@ def image_edit(
             image=images,
             prompt=prompt,
             image_edit_provider_config=image_edit_provider_config,
-            image_edit_optional_request_params=image_edit_request_params,
+            image_edit_optional_request_params={
+                **image_edit_request_params,
+                **(
+                    _tokenhub_passthrough_edit_params(
+                        non_default_params,
+                        extra_body if isinstance(extra_body, dict) else None,
+                        provider_supported_params,
+                        additional_drop_params,
+                    )
+                    if custom_llm_provider == litellm.LlmProviders.TENCENT.value
+                    else {}
+                ),
+            },
             custom_llm_provider=custom_llm_provider,
             litellm_params=litellm_params,
             logging_obj=litellm_logging_obj,

litellm/images/utils.py

@@ -1,6 +1,9 @@
+import base64
+import os
 from collections.abc import Collection, Mapping
 from io import BufferedReader, BytesIO
-from typing import Any, Final, cast, get_type_hints
+from pathlib import Path
+from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
 
 import litellm
 from litellm.litellm_core_utils.token_counter import get_image_type
@@ -16,7 +19,7 @@ class ImageEditRequestUtils:
         image_edit_provider_config: BaseImageEditConfig,
         image_edit_optional_params: ImageEditOptionalRequestParams,
         drop_params: bool | None = None,
-        additional_drop_params: list[str] | None = None,
+        additional_drop_params: Collection[str] | None = None,
     ) -> dict:
         """
         Get optional parameters for the image edit API.
@@ -145,3 +148,36 @@ class ImageEditRequestUtils:
         except Exception:
             # If anything goes wrong, default to PNG
             return FILE_MIME_TYPES[FileType.PNG]
+
+
+@runtime_checkable
+class _SeekableBinaryReader(Protocol):
+    def tell(self) -> int: ...
+
+    def seek(self, offset: int) -> int: ...
+
+    def read(self) -> bytes: ...
+
+
+def _read_image_bytes(image: object) -> bytes:
+    if isinstance(image, bytes):
+        return image
+    if isinstance(image, tuple):
+        return _read_image_bytes(image[1])
+    if isinstance(image, os.PathLike):
+        return Path(image).read_bytes()
+    if isinstance(image, _SeekableBinaryReader):
+        position: Final = image.tell()
+        image.seek(0)
+        data: Final = image.read()
+        image.seek(position)
+        return data
+    raise ValueError(f"Unsupported image type for image edit: {type(image).__name__}")
+
+
+def image_to_data_url(image: object) -> str:
+    if isinstance(image, str):
+        return image
+    image_bytes: Final = _read_image_bytes(image)
+    mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
+    return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"

litellm/integrations/otel/model/spans.py

@@ -338,6 +338,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_ToolTable",
         "LiteLLM_TelemetryReport",
         "LiteLLM_AccessGroupTable",
+        "LiteLLM_AccountingDelivery",
+        "LiteLLM_AccountingEvent",
         "LiteLLM_ClaudeCodePluginTable",
         "LiteLLM_MemoryTable",
         "LiteLLM_AdaptiveRouterState",
@@ -434,6 +436,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
         "reset_budget_windows": PostgresOperation("select", None),
         "write_budget_windows": PostgresOperation("update", None),
         "roll_window_spend_row": PostgresOperation("update", None),
+        "open_window_spend_row": PostgresOperation("upsert", None),
         "seed_window_spend": PostgresOperation("select", None),
         "select_window_spend_rows": PostgresOperation("select", None),
         "commit_window_spend_updates": PostgresOperation("upsert", None),

litellm/integrations/websearch_interception/handler.py

@@ -97,6 +97,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
 
 _RESPONSE_CONTENT_FIELD: Final = "content"
 
+_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
+
 _ResponseT: Final = TypeVar("_ResponseT")
 
 
@@ -363,7 +365,8 @@ class WebSearchInterceptionLogger(CustomLogger):
             get_last_user_message,
         )
 
-        query: Final = get_last_user_message(cast(list[AllMessageValues], messages))
+        last_user_message: Final = get_last_user_message(cast(list[AllMessageValues], messages)) or ""
+        query: Final = last_user_message.removeprefix(_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX).strip()
         if not query:
             return None
 

litellm/litellm_core_utils/README.md

@@ -11,4 +11,4 @@ Core files:
 - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
 - `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
 
-Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
+Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.

litellm/litellm_core_utils/cli_token_utils.py

@@ -353,9 +353,9 @@ def is_cli_token_fresh(
     buffer_hours: float = CLI_TOKEN_FRESHNESS_BUFFER_SECONDS / 3600,
 ) -> bool:
     """Check whether a cached CLI token is still within its expiration window.
-    Used by `lite auth print-token` to fail fast, without a network round trip,
-    once the cached token is past `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce`
-    credential carries its own `expires_at`, which is authoritative when present."""
+    Used by `lite auth print-token` to skip asking the proxy while the cached token
+    is younger than `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce` credential
+    carries its own `expires_at`, which is authoritative when present."""
     from litellm.constants import CLI_JWT_EXPIRATION_HOURS
 
     expires_at: Final = (

litellm/litellm_core_utils/core_helpers.py

@@ -266,6 +266,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "tool_use": "tool_calls",
     "refusal": "content_filter",
     "compaction": "length",
+    "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
     "ERROR_TOXIC": "content_filter",

litellm/litellm_core_utils/get_litellm_params.py

@@ -75,6 +75,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "rpm",
             "itpm",
             "otpm",
+            "tpd",
+            "rpd",
             "use_xai_oauth",
             "chatgpt_auth_file",
             "github_copilot_auth_type",

litellm/litellm_core_utils/litellm_logging.py

@@ -842,6 +842,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self._async_success_enqueued: bool = False
         self._sync_success_scheduled: bool = False
         self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
+        self._stream_failure_claimed: bool = False
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
     def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
@@ -2561,6 +2562,12 @@ class Logging(LiteLLMLoggingBaseClass):
                 },
             )
 
+    def claim_stream_failure_report(self) -> bool:
+        if self._stream_failure_claimed:
+            return False
+        self._stream_failure_claimed = True
+        return True
+
     def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
         """Bill a fully streamed response on the failure log when a post-call hook rejects it."""
         usage: Final = getattr(assembled, "usage", None)
@@ -2751,7 +2758,7 @@ class Logging(LiteLLMLoggingBaseClass):
         if entry is None or not is_str_keyed_dict(litellm_params):
             return
         metadata: Final[object] = litellm_params.get("metadata")
-        bucket: Final = metadata if is_str_keyed_dict(metadata) else {}
+        bucket: Final = {**metadata} if is_str_keyed_dict(metadata) else {}
         litellm_params["metadata"] = bucket
         with_guardrail_entry(bucket, entry)
 
@@ -3474,7 +3481,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             start_time=start_time,
                             end_time=end_time,
                             print_verbose=print_verbose,
-                            level=LogfireLevel.INFO.value,
+                            level=LogfireLevel.INFO,
                         )
 
                     if callback == "lunary" and lunaryLogger is not None:
@@ -4435,7 +4442,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             response_obj=result,
                             start_time=start_time,
                             end_time=end_time,
-                            level=LogfireLevel.ERROR.value,
+                            level=LogfireLevel.ERROR,
                             print_verbose=print_verbose,
                         )
 

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from datetime import datetime, timezone, tzinfo
 from types import MappingProxyType
-from typing import Final, Literal, TypedDict, cast
+from typing import Final, Literal, Protocol, TypedDict, cast
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from typing_extensions import ReadOnly
@@ -2085,6 +2085,41 @@ def calculate_image_response_web_search_cost(
     )
 
 
+class _ModelPricedImageCostCalculator(Protocol):
+    def __call__(self, model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: ...
+
+
+def _model_priced_image_cost_calculator(
+    custom_llm_provider: str | None, model: str
+) -> _ModelPricedImageCostCalculator | None:
+    match custom_llm_provider:
+        case litellm.LlmProviders.RECRAFT.value:
+            from litellm.llms.recraft.cost_calculator import cost_calculator as recraft_image_cost_calculator
+
+            return recraft_image_cost_calculator
+        case litellm.LlmProviders.AIML.value:
+            from litellm.llms.aiml.image_generation.cost_calculator import cost_calculator as aiml_image_cost_calculator
+
+            return aiml_image_cost_calculator
+        case litellm.LlmProviders.COMETAPI.value:
+            from litellm.llms.cometapi.image_generation.cost_calculator import (
+                cost_calculator as cometapi_image_cost_calculator,
+            )
+
+            return cometapi_image_cost_calculator
+        case litellm.LlmProviders.RUNWAYML.value:
+            from litellm.llms.runwayml.cost_calculator import cost_calculator as runwayml_image_cost_calculator
+
+            return runwayml_image_cost_calculator
+        case litellm.LlmProviders.TENCENT.value:
+            from litellm.llms.tencent.cost_calculator import image_cost_calculator as tencent_image_cost_calculator
+            from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
+
+            return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
+        case _:
+            return None
+
+
 class CostCalculatorUtils:
     @staticmethod
     def call_type_has_image_response(call_type: str) -> bool:
@@ -2131,9 +2166,6 @@ class CostCalculatorUtils:
         from litellm.llms.gemini.image_generation.cost_calculator import (
             cost_calculator as gemini_image_cost_calculator,
         )
-        from litellm.llms.recraft.cost_calculator import (
-            cost_calculator as recraft_image_cost_calculator,
-        )
         from litellm.llms.vertex_ai.image_generation.cost_calculator import (
             cost_calculator as vertex_ai_image_cost_calculator,
         )
@@ -2146,7 +2178,10 @@ class CostCalculatorUtils:
         )
         resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
         pricing: Final = deployment_pricing(model_info)
+        model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
 
+        if model_priced_calculator is not None:
+            return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
         if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
             if isinstance(completion_response, ImageResponse):
                 return vertex_ai_image_cost_calculator(
@@ -2164,36 +2199,6 @@ class CostCalculatorUtils:
                     optional_params=optional_params,
                 )
             raise TypeError("completion_response must be of type ImageResponse for bedrock image cost calculation")
-        elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
-            from litellm.llms.recraft.cost_calculator import (
-                cost_calculator as recraft_image_cost_calculator,
-            )
-
-            return recraft_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
-        elif custom_llm_provider == litellm.LlmProviders.AIML.value:
-            from litellm.llms.aiml.image_generation.cost_calculator import (
-                cost_calculator as aiml_image_cost_calculator,
-            )
-
-            return aiml_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
-        elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
-            from litellm.llms.cometapi.image_generation.cost_calculator import (
-                cost_calculator as cometapi_image_cost_calculator,
-            )
-
-            return cometapi_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
         elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
             if call_type in (
                 CallTypes.image_edit.value,
@@ -2237,16 +2242,6 @@ class CostCalculatorUtils:
                 optional_params=optional_params,
                 model_info=pricing,
             )
-        elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
-            from litellm.llms.runwayml.cost_calculator import (
-                cost_calculator as runwayml_image_cost_calculator,
-            )
-
-            return runwayml_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
         elif (
             custom_llm_provider == litellm.LlmProviders.OPENAI.value
             or custom_llm_provider == litellm.LlmProviders.AZURE.value

litellm/litellm_core_utils/safe_json_dumps.py

@@ -34,7 +34,7 @@ def safe_json_structure(
     def _transform(key: str | None, value: str) -> str:
         return value if value_transform is None else value_transform(key, value)
 
-    def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
+    def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
         # Check for maximum depth.
         if depth > max_depth:
             return "MaxDepthExceeded"
@@ -48,32 +48,32 @@ def safe_json_structure(
         if id(obj) in seen:
             return "CircularReference Detected"
         seen.add(id(obj))
-        result: dict | list | tuple | set | str
         if isinstance(obj, dict):
-            result = {}
-            for k, v in obj.items():
-                if isinstance(k, (str)):
-                    clean_k = k.replace("\x00", "") if "\x00" in k else k
-                    result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
+            mapping_result: Final = {
+                (clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
+                for k, v in obj.items()
+                if isinstance(k, str)
+            }
             seen.remove(id(obj))
-            return result
+            return mapping_result
         elif isinstance(obj, list):
-            result = [_serialize(item, seen, depth + 1, key) for item in obj]
+            list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
             seen.remove(id(obj))
-            return result
+            return list_result
         elif isinstance(obj, tuple):
-            result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
+            tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
             seen.remove(id(obj))
-            return result
+            return tuple_result
         elif isinstance(obj, set):
-            result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
+            sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
+            set_result: Final = sorted(sortable_values)
             seen.remove(id(obj))
-            return result
+            return set_result
         elif isinstance(obj, BaseModel):
             dumped: Final = obj.model_dump()
-            result = _serialize(dumped, seen, depth + 1, key)
+            model_result: Final = _serialize(dumped, seen, depth + 1, key)
             seen.remove(id(obj))
-            return result
+            return model_result
         else:
             # Fall back to string conversion for non-serializable objects.
             try:

litellm/litellm_core_utils/streaming_handler.py

@@ -1595,6 +1595,12 @@ class CustomStreamWrapper:
         else:  # openai / azure chat model
             if isinstance(chunk, ChatCompletionChunk):
                 self._openai_handler_receives_chat_chunks = True
+                if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
+                    raise litellm.InternalServerError(
+                        message="Upstream generation aborted (finish_reason=abort)",
+                        model=chunk.model,
+                        llm_provider=self.custom_llm_provider,
+                    )
             if self.custom_llm_provider in [
                 LlmProviders.AZURE.value,
                 LlmProviders.AZURE_AI.value,

litellm/llms/aiohttp_openai/chat/transformation.py

@@ -82,4 +82,6 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
         model_response.model = _json_response.get("model")
         model_response.object = _json_response.get("object")
         model_response.system_fingerprint = _json_response.get("system_fingerprint")
+        if "service_tier" in _json_response:
+            return model_response.model_copy(update={"service_tier": _json_response.get("service_tier")})
         return model_response

litellm/llms/anthropic/messages/caller_model_checks.py

@@ -268,6 +268,7 @@ async def caller_within_model_budget(
                 user_id=user_id,
                 user_model_max_budget=user_model_max_budget,
                 model=model,
+                team_id=user_api_key_auth.team_id,
             )
         except litellm.BudgetExceededError:
             return "denied"
@@ -309,6 +310,7 @@ async def caller_within_model_budget(
                 end_user_id=end_user_id,
                 end_user_model_max_budget=end_user_model_max_budget,
                 model=model,
+                team_id=user_api_key_auth.team_id,
             )
         except litellm.BudgetExceededError:
             return "denied"

litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py

@@ -2,15 +2,16 @@
 ## Translates OpenAI call to Anthropic `/v1/messages` format
 import asyncio
 from collections import deque
-from collections.abc import AsyncGenerator, Iterator, Mapping
+from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final
 
+import anyio
 from pydantic import ConfigDict, field_validator
 
 from litellm import verbose_logger
 from litellm._logging import redact_internal_details_from_client_message
 from litellm._uuid import uuid
-from litellm.exceptions import MidStreamFallbackError
+from litellm.exceptions import APIError, MidStreamFallbackError
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     encrypted_reasoning_signature,
 )
@@ -22,6 +23,7 @@ from litellm.llms.anthropic.messages.native.utils import (
 from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
 from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
 from litellm.responses.streaming_iterator import (
+    map_stream_error_to_exception,
     partial_responses_incomplete_event_on_disconnect,
     stream_error_status_and_message,
 )
@@ -130,10 +132,12 @@ class AnthropicResponsesStreamWrapper:
     ) -> None:
         self.responses_stream = responses_stream
         self.model = model
+        self.litellm_logging_obj = litellm_logging_obj
         self.safeguards_evaluator: Final = safeguards_evaluator
         self._message_id: str = f"msg_{uuid.uuid4()}"
         if litellm_logging_obj is not None:
             litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
+        self._unreported_failure: Exception | None = None
         self._current_block_index: int = -1
         # Map item_id -> content_block_index so we can stop the right block later
         self._item_id_to_block_index: dict[str, int] = {}
@@ -377,16 +381,24 @@ class AnthropicResponsesStreamWrapper:
 
         if event_type == "response.failed":
             failed: Final = _FailedResponseEvent.model_validate(event)
-            status_code, message = stream_error_status_and_message(
-                failed.response.error if failed.response is not None else None
-            )
+            upstream_error: Final = failed.response.error if failed.response is not None else None
+            status_code, message = stream_error_status_and_message(upstream_error)
             verbose_logger.error(
                 "AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed (%s): %s",
                 self.model,
                 status_code,
                 message,
             )
-            self._fail_stream(status_code, message)
+            self._fail_stream(
+                status_code,
+                message,
+                failure=map_stream_error_to_exception(
+                    upstream_error,
+                    self.model,
+                    getattr(self.responses_stream, "custom_llm_provider", None) or "",
+                    logging_obj=self.litellm_logging_obj,
+                ),
+            )
             return
 
         # ---- response completed -> message_delta + message_stop ----
@@ -439,10 +451,36 @@ class AnthropicResponsesStreamWrapper:
             self._sent_message_stop = True
             return
 
-    def _fail_stream(self, status_code: int, message: str) -> None:
+    def _fail_stream(self, status_code: int, message: str, failure: Exception | None = None) -> None:
         self._stream_failed = True
+        self._unreported_failure = failure or APIError(
+            status_code=status_code,
+            message=message,
+            llm_provider=getattr(self.responses_stream, "custom_llm_provider", None),
+            model=self.model,
+        )
         self._chunk_queue.append(_anthropic_error_chunk(status_code, message))
 
+    async def _report_unreported_failure(self) -> None:
+        failure: Final = self._unreported_failure
+        if failure is None:
+            return
+        self._unreported_failure = None
+        on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr(
+            self.litellm_logging_obj, "_on_detached_stream_failure", None
+        )
+        if on_failure is None:
+            return
+        try:
+            with anyio.CancelScope(shield=True):
+                await on_failure(failure)
+        except Exception as hook_failure:  # noqa: BLE001  # a failing proxy hook must not hide the error event from the client
+            verbose_logger.warning(
+                "AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)",
+                type(hook_failure).__name__,
+                hook_failure,
+            )
+
     def __aiter__(self) -> "AnthropicResponsesStreamWrapper":
         return self
 
@@ -450,6 +488,7 @@ class AnthropicResponsesStreamWrapper:
         if self._chunk_queue:
             return self._chunk_queue.popleft()
         if self._stream_failed:
+            await self._report_unreported_failure()
             raise StopAsyncIteration
 
         if not self._sent_message_start:
@@ -478,7 +517,7 @@ class AnthropicResponsesStreamWrapper:
             verbose_logger.exception(
                 "AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed", self.model
             )
-            self._fail_stream(*_failure_status_and_message(e))
+            self._fail_stream(*_failure_status_and_message(e), failure=_original_failure(e))
 
         if not self._chunk_queue and not self._sent_message_stop and not self._stream_failed:
             verbose_logger.error(
@@ -498,8 +537,14 @@ class AnthropicResponsesStreamWrapper:
 
     async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
         safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
-        async for chunk in self:
-            yield anthropic_sse_frame(await safeguard_results.observe(chunk))
+        try:
+            async for chunk in self:
+                yield anthropic_sse_frame(await safeguard_results.observe(chunk))
+        finally:
+            from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt
+
+            if not closing_superseded_attempt():
+                await self._report_unreported_failure()
 
     def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
         return partial_responses_incomplete_event_on_disconnect(self.responses_stream)

litellm/llms/base_llm/files/transformation.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Union
+from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Final, Union
 
 import httpx
 from openai.types.file_deleted import FileDeleted
@@ -38,6 +39,22 @@ else:
     UserAPIKeyAuth = Any
 
 
+def _routes_nowhere(_model_name: str) -> bool:
+    return False
+
+
+@dataclass(frozen=True, slots=True)
+class FileListModelScope:
+    model_mapping_keys: frozenset[str]
+    routes_stored_model_name: Callable[[str], bool] = _routes_nowhere
+
+    def holds(self, model_mapping_keys: Iterable[str], stored: bool) -> bool:
+        mapped_keys: Final = frozenset(model_mapping_keys)
+        return not self.model_mapping_keys.isdisjoint(mapped_keys) or (
+            stored and any(map(self.routes_stored_model_name, mapped_keys))
+        )
+
+
 class BaseFileUploadStream(ABC):
     """Re-iterable request body that yields an upload's bytes lazily.
 
@@ -274,6 +291,8 @@ class BaseFileEndpoints(ABC):
         user_api_key_dict: UserAPIKeyAuth,
         limit: int | None = None,
         after: str | None = None,
+        model_scope: FileListModelScope | None = None,
+        before: str | None = None,
         **data: dict,
     ) -> FileListPage:
         pass

litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py

@@ -10,6 +10,8 @@ import time
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -34,6 +36,15 @@ else:
     LiteLLMLoggingObj = Any
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _PegasusResponse(TypedDict):
+    message: ReadOnly[NotRequired[str | None]]
+    finishReason: ReadOnly[NotRequired[str | None]]
+
+
+_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
+
+
 class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
     """
     Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
@@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
         ModelResponse with choices[0].message.content and finish_reason
         """
         try:
-            completion_response: Final = raw_response.json()
+            completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
         except Exception as e:
             raise BedrockError(
                 message=f"Error parsing response: {raw_response.text}, error: {e}",
@@ -224,7 +235,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
         message_content: Final = completion_response.get("message", "")
 
         # Extract finish reason and map to LiteLLM format
-        finish_reason_raw: Final = completion_response.get("finishReason", "stop")
+        finish_reason_raw: Final = completion_response.get("finishReason") or "stop"
         finish_reason: Final = map_finish_reason(finish_reason_raw)
 
         # Set the response content

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -1,11 +1,12 @@
 import copy
 import json
 import time
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, cast, get_args
 
 import httpx
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 
 _GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
 
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CohereGeneration(TypedDict):
+    text: ReadOnly[str | None]
+    finish_reason: ReadOnly[str | None]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CohereInvokeResponse(TypedDict):
+    text: NotRequired[ReadOnly[str | None]]
+    generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _LlamaInvokeResponse(TypedDict):
+    generation: ReadOnly[str | None]
+
+
+_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
+_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
+
 _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
     "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
 )
@@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         outputText: str | None = None
         try:
             if provider == "cohere":
-                if "text" in completion_response:
-                    outputText = completion_response["text"]
-                elif "generations" in completion_response:
-                    outputText = completion_response["generations"][0]["text"]
+                cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
+                if "text" in cohere_response:
+                    outputText = cohere_response["text"]
+                elif "generations" in cohere_response:
+                    outputText = cohere_response["generations"][0]["text"]
                     model_response.choices[0].finish_reason = map_finish_reason(
-                        completion_response["generations"][0]["finish_reason"]
+                        cohere_response["generations"][0]["finish_reason"] or "stop"
                     )
             elif provider == "anthropic":
                 return litellm.AmazonAnthropicClaudeConfig().transform_response(
@@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
             elif provider == "ai21":
                 outputText = completion_response.get("completions")[0].get("data").get("text")
             elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
-                outputText = completion_response["generation"]
+                outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
             elif provider == "mistral":
                 outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
             else:  # amazon titan

litellm/llms/bedrock/realtime/transformation.py

@@ -53,6 +53,7 @@ from litellm.types.realtime import (
     RealtimeResponseTypedDict,
 )
 
+_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
 _JSON_VALUE: Final = TypeAdapter(object)
 _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
 
@@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             List of Bedrock format messages (JSON strings)
         """
         try:
-            json_message: Final[dict[str, JsonValue]] = json.loads(message)
+            json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
         except json.JSONDecodeError:
             verbose_logger.warning("Invalid JSON message: %s", message[:200])
             return []

litellm/llms/custom_httpx/aiohttp_transport.py

@@ -6,6 +6,7 @@ import ssl
 import sys
 import typing
 import urllib.request
+import weakref
 from collections.abc import Callable, Generator
 from typing import ClassVar, Final
 
@@ -109,6 +110,24 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
         raise mapped_exc(message) from exc
 
 
+def _release_transport_anchor(_transport: object) -> None:
+    """Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
+
+
+def _anchor_transport_to(stream: "AiohttpResponseStream", transport: object) -> None:
+    """Keep the transport alive for as long as the stream can still read.
+
+    The transport finalizer closes the session it owns, and nothing on the path from a
+    live response back to that session references the transport, so a client dropped
+    mid-stream would close the session under the body. ``weakref.finalize`` holds the
+    transport in its own registry rather than on the stream, which keeps the transport
+    out of the response's reference cycle (an exception carrying the response, for one)
+    so it is finalized by refcount once the anchor drops and can still schedule an async
+    close, instead of being collected in the same cyclic pass as the session it closes.
+    """
+    weakref.finalize(stream, _release_transport_anchor, transport)
+
+
 class AiohttpResponseStream(httpx.AsyncByteStream):
     CHUNK_SIZE = 1024 * 16
 
@@ -463,10 +482,12 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
                 # Re-raise if it's a different RuntimeError
                 raise
 
+        stream: Final = AiohttpResponseStream(response)
+        _anchor_transport_to(stream, self)
         return httpx.Response(
             status_code=response.status,
             headers=response.headers,
-            stream=AiohttpResponseStream(response),
+            stream=stream,
             request=request,
         )
 

litellm/llms/dashscope/embed/transformation.py

@@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import ensure_api_route
@@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
 DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _DashScopeUsage(TypedDict, total=False):
+    prompt_tokens: NotRequired[ReadOnly[int]]
+    total_tokens: NotRequired[ReadOnly[int]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _DashScopeEmbeddingResponse(TypedDict, total=False):
+    data: NotRequired[ReadOnly[list[object]]]
+    model: NotRequired[ReadOnly[str | None]]
+    usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
+    id: NotRequired[ReadOnly[object]]
+
+
+_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
+
+
 class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
     """
     Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
@@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
                 message=message,
             )
 
+        parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
         model_response.object = "list"
-        model_response.data = response_json.get("data", [])
-        model_response.model = response_json.get("model", model)
+        model_response.data = parsed_response.get("data", [])
+        model_response.model = parsed_response.get("model", model)
 
-        usage: Final = response_json.get("usage") or {}
+        usage: Final = parsed_response.get("usage") or {}
         prompt_tokens: Final = usage.get("prompt_tokens", 0)
         total_tokens: Final = usage.get("total_tokens", prompt_tokens)
         setattr(
@@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
             ),
         )
 
-        if "id" in response_json:
-            setattr(model_response, "id", response_json["id"])
+        if "id" in parsed_response:
+            setattr(model_response, "id", parsed_response["id"])
 
         return model_response
 

litellm/llms/databricks/streaming_utils.py

@@ -1,6 +1,8 @@
 import json
 from typing import Final
 
+from pydantic import ConfigDict, TypeAdapter
+
 import litellm
 from litellm import verbose_logger
 from litellm.types.llms.openai import (
@@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
     )
 
 
+_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
+
 class ModelResponseIterator:
     def __init__(self, streaming_response, sync_stream: bool):
         self.streaming_response = streaming_response
@@ -99,7 +104,7 @@ class ModelResponseIterator:
             self.response_iterator = self.streaming_response
         chunk: Final = self._next_sse_data()
         try:
-            json_chunk: Final = json.loads(chunk)
+            json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
             return self.chunk_parser(chunk=json_chunk)
         except ValueError as e:
             verbose_logger.debug(
@@ -126,7 +131,7 @@ class ModelResponseIterator:
         if chunk == "[DONE]":
             raise StopAsyncIteration
         try:
-            json_chunk: Final = json.loads(chunk)
+            json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
             return self.chunk_parser(chunk=json_chunk)
         except ValueError as e:
             verbose_logger.debug(

litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py

@@ -4,13 +4,14 @@ from typing import Final
 
 from httpx._types import RequestFiles
 
+from litellm.images.utils import image_to_data_url
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.images.main import ImageEditOptionalRequestParams
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import FileTypes
 
-from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url
+from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig
 
 FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth"
 SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size")
@@ -68,7 +69,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
         )
         request_body: Final[dict[str, object]] = {  # mutable-ok: base class contract returns a dict
             "prompt": prompt,
-            "image_url": to_data_url(next(iter(images))),
+            "image_url": image_to_data_url(next(iter(images))),
             **provider_params,
         }
         return request_body, ()

litellm/llms/fal_ai/image_edit/transformation.py

@@ -1,15 +1,12 @@
-import base64
-import os
 from collections.abc import Mapping
-from pathlib import Path
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
+from typing import TYPE_CHECKING, Final
 
 import httpx
 from httpx._types import RequestFiles
 from pydantic import ConfigDict, TypeAdapter
 
-from litellm.images.utils import ImageEditRequestUtils
+from litellm.images.utils import image_to_data_url
 from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
 from litellm.llms.fal_ai.image_generation.gpt_image_2_transformation import (
     map_gpt_image_quality,
@@ -39,39 +36,6 @@ PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType(
 )
 
 
-@runtime_checkable
-class _SeekableBinaryReader(Protocol):
-    def tell(self) -> int: ...
-
-    def seek(self, offset: int) -> int: ...
-
-    def read(self) -> bytes: ...
-
-
-def _read_image_bytes(image: object) -> bytes:
-    if isinstance(image, bytes):
-        return image
-    if isinstance(image, tuple):
-        return _read_image_bytes(image[1])
-    if isinstance(image, os.PathLike):
-        return Path(image).read_bytes()
-    if isinstance(image, _SeekableBinaryReader):
-        position: Final = image.tell()
-        image.seek(0)
-        data: Final = image.read()
-        image.seek(position)
-        return data
-    raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}")
-
-
-def to_data_url(image: object) -> str:
-    if isinstance(image, str):
-        return image
-    image_bytes: Final = _read_image_bytes(image)
-    mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
-    return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
-
-
 def _first(value: object) -> object:
     return value[0] if isinstance(value, list) and value else value
 
@@ -146,14 +110,14 @@ class FalAIImageEditConfig(BaseImageEditConfig):
             raise ValueError("Fal AI image edit requires at least one input image")
         mask: Final = _first(image_edit_optional_request_params.get("mask"))
         mask_field: Final[Mapping[str, str]] = (
-            MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({})
+            MappingProxyType({"mask_url": image_to_data_url(mask)}) if mask is not None else MappingProxyType({})
         )
         provider_params: Final[Mapping[str, object]] = MappingProxyType(
             {key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
         )
         request_body: Final[dict[str, object]] = {  # mutable-ok: base class contract returns a dict
             "prompt": prompt,
-            "image_urls": tuple(to_data_url(img) for img in images),
+            "image_urls": tuple(image_to_data_url(img) for img in images),
             **mask_field,
             **provider_params,
         }

litellm/llms/gigachat/authenticator.py

@@ -12,6 +12,8 @@ from types import MappingProxyType
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
 from litellm.caching.caching import InMemoryCache
@@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache()
 _NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _GigaChatTokenResponse(TypedDict):
+    tok: NotRequired[ReadOnly[str | None]]
+    access_token: NotRequired[ReadOnly[str | None]]
+    exp: NotRequired[ReadOnly[int | float | str | None]]
+    expires_at: NotRequired[ReadOnly[int | float | str | None]]
+
+
+_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse)
+
+
 class GigaChatAuthError(BaseLLMException):
     """GigaChat authentication error."""
 
@@ -230,7 +243,7 @@ async def _request_token_async(
 
 def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
     """Parse OAuth token response."""
-    data: Final = response.json()
+    data: Final = _TOKEN_RESPONSE.validate_python(response.json())
 
     # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
     access_token: Final = data.get("tok") or data.get("access_token")

litellm/llms/openai/chat/gpt_transformation.py

@@ -76,6 +76,7 @@ else:
 
 _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
 _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
 _DECODED_JSON: Final = TypeAdapter(object)
 
 
@@ -958,8 +959,13 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
                     status_code=error_status_code,
                     message=error_message,
                 )
-            choices = chunk.get("choices", [])
-            choices = self._map_reasoning_to_reasoning_content(choices)
+            raw_choices: Final = chunk.get("choices") or []
+            if any(choice.get("finish_reason") == "abort" for choice in _CHOICE_MAPPINGS.validate_python(raw_choices)):
+                raise OpenAIError(
+                    status_code=500,
+                    message="Upstream generation aborted (finish_reason=abort)",
+                )
+            choices: Final = self._map_reasoning_to_reasoning_content(raw_choices)
             fingerprint: Final = _OBJECT_MAPPING.validate_python(chunk).get("system_fingerprint")
 
             kwargs: Final[dict[str, Any]] = {

litellm/llms/openai/realtime/handler.py

@@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
 """
 
 import ssl
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
 
 from litellm._logging import redact_string, verbose_logger
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
@@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
 from ..common_utils import is_openai_backed_api_base
 from ..openai import OpenAIChatCompletion
 
+if TYPE_CHECKING:
+    from fastapi import WebSocket
+
 
 class OpenAIRealtime(OpenAIChatCompletion):
     """
@@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
     async def async_realtime(
         self,
         model: str,
-        websocket: Any,
+        websocket: "WebSocket",
         logging_obj: LiteLLMLogging,
         api_base: str | None = None,
         api_key: str | None = None,

litellm/llms/perplexity/embedding/transformation.py

@@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses.
 """
 
 import base64
+import builtins
 import struct
-from collections.abc import Mapping
-from typing import Final
+from collections.abc import Mapping, Sequence
+from typing import Final, Literal
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import join_api_base
@@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
 _INT: Final = TypeAdapter(int)
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _PerplexityEmbeddingResponse(TypedDict, total=False):
+    model: ReadOnly[str | None]
+    object: ReadOnly[Literal["list"]]
+    data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]]
+    usage: ReadOnly[builtins.object]
+
+
+_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse)
+
+
 class PerplexityEmbeddingError(BaseLLMException):
     def __init__(
         self,
@@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
         litellm_params: dict = {},
     ) -> EmbeddingResponse:
         try:
-            raw_response_json: Final = raw_response.json()
+            raw_response_json: Final[object] = raw_response.json()
         except Exception:
             raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code)
 
-        model_response.model = raw_response_json.get("model", model)
-        model_response.object = raw_response_json.get("object", "list")
+        response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python(
+            raw_response_json
+        )
 
-        raw_data: Final = raw_response_json.get("data", [])
+        model_response.model = response_data.get("model", model)
+        model_response.object = response_data.get("object", "list")
+
+        raw_data: Final = response_data.get("data", [])
         decoded_data: Final[list[dict[str, object]]] = []
         for item in raw_data:
             decoded_item = dict(_JSON_OBJECT.validate_python(item))
@@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
             decoded_data.append(decoded_item)
         model_response.data = decoded_data
 
-        usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
+        usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {}))
         usage: Final = Usage(
             prompt_tokens=_INT.validate_python(
                 usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0

litellm/llms/tencent/cost_calculator.py

@@ -1,6 +1,49 @@
-from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
-from litellm.types.utils import Usage
+from typing import Final
+
+from litellm.litellm_core_utils.llm_cost_calc.utils import (
+    calculate_image_response_cost_from_usage,
+    flat_image_cost,
+    generic_cost_per_token,
+    resolve_image_model_info,
+)
+from litellm.types.utils import ImageResponse, ModelInfo, Usage
 
 
 def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
     return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")
+
+
+def _deployment_image_token_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
+    if model_info is None or model_info.get("output_cost_per_image_token") is not None:
+        return model_info
+    output_cost_per_token: Final = model_info.get("output_cost_per_token")
+    if output_cost_per_token is None:
+        return model_info
+    priced: Final[ModelInfo] = {**model_info, "output_cost_per_image_token": output_cost_per_token}
+    return priced
+
+
+def _deployment_prices_per_image_only(model_info: ModelInfo | None) -> bool:
+    return (
+        model_info is not None
+        and model_info.get("output_cost_per_image") is not None
+        and model_info.get("output_cost_per_image_token") is None
+        and model_info.get("output_cost_per_token") is None
+    )
+
+
+def image_cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
+    resolved_model_info: Final = resolve_image_model_info(
+        model=model, custom_llm_provider="tencent", model_info=_deployment_image_token_pricing(model_info)
+    )
+    if _deployment_prices_per_image_only(model_info):
+        return flat_image_cost(resolved_model_info, image_response)
+    token_based_cost: Final = calculate_image_response_cost_from_usage(
+        model=model,
+        image_response=image_response,
+        custom_llm_provider="tencent",
+        model_info=resolved_model_info,
+    )
+    if token_based_cost is not None:
+        return token_based_cost
+    return flat_image_cost(resolved_model_info, image_response)

litellm/llms/tencent/image_edit/transformation.py

@@ -0,0 +1,147 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+import httpx
+from httpx._types import RequestFiles
+from pydantic import TypeAdapter
+from typing_extensions import override
+
+from litellm.images.utils import image_to_data_url
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
+from litellm.llms.tencent.image_generation.transformation import (
+    TOKENHUB_IMAGE_PARAMS,
+    tokenhub_image_headers,
+    tokenhub_image_params,
+    tokenhub_image_response,
+    tokenhub_image_url,
+)
+from litellm.types.images.main import ImageEditOptionalRequestParams
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import ImageResponse
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_IMAGES: Final = TypeAdapter(tuple[object, ...])
+_FORM_BOOLEANS: Final = MappingProxyType({"true": True, "false": False})
+_OPENAI_PARAMS: Final = frozenset(TOKENHUB_IMAGE_PARAMS)
+
+_FormEntries: TypeAlias = tuple[tuple[tuple[str, ...], object], ...]
+
+
+def _form_scalar(value: object) -> object:
+    match value:
+        case str() if value.lower() in _FORM_BOOLEANS:
+            return _FORM_BOOLEANS[value.lower()]
+        case str() if value.removeprefix("-").isdigit():
+            return int(value)
+        case _:
+            return value
+
+
+def _form_path(key: str) -> tuple[str, ...]:
+    root, bracket, rest = key.partition("[")
+    if not bracket or not rest.endswith("]"):
+        return (key,)
+    return (root, *rest.removesuffix("]").split("]["))
+
+
+def _nested_form_value(entries: _FormEntries) -> object:
+    match entries:
+        case (((), value),):
+            return _form_scalar(value)
+        case _:
+            return _nested_form_fields(tuple(entry for entry in entries if entry[0]))
+
+
+def _nested_form_fields(entries: _FormEntries) -> Mapping[str, object]:
+    roots: Final = tuple(dict.fromkeys(path[0] for path, _ in entries))
+    return {
+        root: _nested_form_value(tuple((path[1:], value) for path, value in entries if path[0] == root))
+        for root in roots
+    }
+
+
+def _tokenhub_form_params(params: Mapping[str, object]) -> Mapping[str, object]:
+    tokenhub_entries: Final = tuple(
+        (_form_path(key), value) for key, value in params.items() if key not in _OPENAI_PARAMS
+    )
+    return {
+        **{key: value for key, value in params.items() if key in _OPENAI_PARAMS},
+        **_nested_form_fields(tokenhub_entries),
+    }
+
+
+class TencentImageEditConfig(BaseImageEditConfig):
+    @override
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: mirrors override contract
+        return list(TOKENHUB_IMAGE_PARAMS)
+
+    @override
+    def map_openai_params(
+        self,
+        image_edit_optional_params: ImageEditOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: mirrors override contract
+        return dict(tokenhub_image_params(image_edit_optional_params, model, drop_params))
+
+    @override
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        api_key: str | None = None,
+        litellm_params: Mapping[str, object] | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: mirrors override contract
+        return dict(tokenhub_image_headers(headers, api_key, model))
+
+    @override
+    def get_complete_url(
+        self,
+        model: str,
+        api_base: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> str:
+        return tokenhub_image_url(api_base)
+
+    @override
+    def use_multipart_form_data(self) -> bool:
+        return False
+
+    @override
+    def transform_image_edit_request(
+        self,
+        model: str,
+        prompt: str | None,
+        image: object,
+        image_edit_optional_request_params: Mapping[str, object],
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[dict[str, object], RequestFiles]:  # mutable-ok: mirrors override contract
+        images: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
+        return {
+            "model": model,
+            "prompt": prompt,
+            "images": tuple(image_to_data_url(img) for img in images if img is not None),
+            **_tokenhub_form_params(image_edit_optional_request_params),
+        }, ()
+
+    @override
+    def transform_image_edit_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "LiteLLMLoggingObj",
+    ) -> ImageResponse:
+        image_response: Final = tokenhub_image_response(raw_response)
+        if image_response is None:
+            raise BaseLLMException(
+                status_code=raw_response.status_code,
+                message=raw_response.text,
+                headers=raw_response.headers,
+            )
+        return image_response

litellm/llms/tencent/image_generation/transformation.py

@@ -0,0 +1,200 @@
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+import httpx
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+from typing_extensions import override
+
+from litellm.exceptions import AuthenticationError, UnsupportedParamsError
+from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.image_generation.transformation import (
+    BaseImageGenerationConfig,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIImageGenerationOptionalParams,
+)
+from litellm.types.utils import (
+    ImageObject,
+    ImageResponse,
+    ImageUsage,
+    ImageUsageInputTokensDetails,
+    LlmProviders,
+)
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+TOKENHUB_API_BASE: Final = "https://tokenhub-intl.tencentcloudmaas.com/v1"
+TOKENHUB_IMAGE_PATH: Final = "/wand/si-image/generation"
+TOKENHUB_IMAGE_MODEL_PREFIX: Final = "seedream-"
+TOKENHUB_IMAGE_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = (
+    "n",
+    "size",
+    "response_format",
+    "output_format",
+    "background",
+    "user",
+)
+
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+
+
+class _TokenHubImage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    url: str | None = None
+    b64_json: str | None = None
+    size: str | None = None
+    output_format: str | None = None
+
+
+class _TokenHubUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    total_tokens: int
+
+
+class _TokenHubImageResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    created: int
+    data: tuple[_TokenHubImage, ...]
+    tokenhub_usage: _TokenHubUsage | None = None
+
+
+def is_tokenhub_image_model(model: str) -> bool:
+    return model.removeprefix(f"{LlmProviders.TENCENT.value}/").startswith(TOKENHUB_IMAGE_MODEL_PREFIX)
+
+
+def tokenhub_image_url(api_base: str | None) -> str:
+    base: Final = api_base or get_secret_str("TENCENT_API_BASE") or TOKENHUB_API_BASE
+    return join_api_base(ensure_api_route(base, "v1"), TOKENHUB_IMAGE_PATH)
+
+
+def tokenhub_image_headers(headers: Mapping[str, str], api_key: str | None, model: str) -> Mapping[str, str]:
+    resolved_api_key: Final = api_key or get_secret_str("TENCENT_API_KEY")
+    if not resolved_api_key:
+        raise AuthenticationError(
+            message="TENCENT_API_KEY is not set. Set it in the environment or pass api_key.",
+            llm_provider="tencent",
+            model=model,
+        )
+    return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
+
+
+def tokenhub_image_params(params: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, object]:
+    n: Final = params.get("n")
+    if n is not None and n != 1 and not drop_params:
+        raise UnsupportedParamsError(
+            status_code=400,
+            message=f"Tencent TokenHub returns one image per request, so n={n} is not supported for {model}. "
+            + "To send n=1 instead, set `litellm.drop_params = True`.",
+        )
+    return {
+        key: value
+        for key, value in params.items()
+        if key != "n" and value is not None and not (key == "background" and value == "auto")
+    }
+
+
+def tokenhub_image_response(raw_response: httpx.Response) -> ImageResponse | None:
+    try:
+        parsed: Final = _TokenHubImageResponse.model_validate_json(raw_response.content)
+    except ValidationError:
+        return None
+    total_tokens: Final = parsed.tokenhub_usage.total_tokens if parsed.tokenhub_usage is not None else 0
+    first_image: Final = parsed.data[0] if parsed.data else _TokenHubImage()
+    return ImageResponse(
+        created=parsed.created,
+        data=[ImageObject(url=image.url, b64_json=image.b64_json) for image in parsed.data],
+        size=first_image.size,
+        output_format=first_image.output_format,
+        usage=ImageUsage(
+            input_tokens=0,
+            input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0),
+            output_tokens=total_tokens,
+            total_tokens=total_tokens,
+        ),
+    )
+
+
+class TencentImageGenerationConfig(BaseImageGenerationConfig):
+    @override
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIImageGenerationOptionalParams]:  # mutable-ok: mirrors override contract
+        return list(TOKENHUB_IMAGE_PARAMS)
+
+    @override
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: mirrors override contract
+        return {**optional_params, **tokenhub_image_params(non_default_params, model, drop_params)}
+
+    @override
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        return tokenhub_image_url(api_base)
+
+    @override
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: mirrors override contract
+        return dict(tokenhub_image_headers(headers, api_key, model))
+
+    @override
+    def transform_image_generation_request(
+        self,
+        model: str,
+        prompt: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        headers: Mapping[str, str],
+    ) -> dict[str, object]:  # mutable-ok: mirrors override contract
+        extra_body: Final = _JSON_OBJECT.validate_python(optional_params.get("extra_body") or {})
+        params: Final = {key: value for key, value in optional_params.items() if key != "extra_body"}
+        return {"model": model, "prompt": prompt, **params, **extra_body}
+
+    @override
+    def transform_image_generation_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: ImageResponse,
+        logging_obj: "LiteLLMLoggingObj",
+        request_data: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        encoding: object,
+        api_key: str | None = None,
+        json_mode: bool | None = None,
+    ) -> ImageResponse:
+        image_response: Final = tokenhub_image_response(raw_response)
+        if image_response is None:
+            raise BaseLLMException(
+                status_code=raw_response.status_code,
+                message=raw_response.text,
+                headers=raw_response.headers,
+            )
+        return image_response

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -2743,10 +2743,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 chunk_indices = support.get("groundingChunkIndices", [])
 
                 if start_index is not None and end_index is not None and chunk_indices:
-                    # Use the first chunk's URL for the annotation
-                    first_chunk_idx = chunk_indices[0]
-                    if first_chunk_idx in chunk_to_uri_map:
-                        uri_info = chunk_to_uri_map[first_chunk_idx]
+                    for chunk_index in chunk_indices:
+                        if chunk_index not in chunk_to_uri_map:
+                            continue
+                        uri_info = chunk_to_uri_map[chunk_index]
 
                         url_citation: ChatCompletionAnnotationURLCitation = {
                             "start_index": start_index,

litellm/llms/vertex_ai/vertex_gemma_models/transformation.py

@@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI
 """
 
 from collections.abc import Callable
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
 
 import httpx
-from pydantic import ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError
+from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
@@ -32,6 +33,17 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
 
+class _PredictionsEnvelope(TypedDict):
+    predictions: NotRequired[ReadOnly[dict[str, object]]]
+
+
+_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope)
+_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter(
+    _PredictionsEnvelopeType,
+    config=ConfigDict(strict=True, extra="allow"),
+)
+
+
 def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None:
     try:
         return VertexGemmaContainerError.model_validate(predictions)
@@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
 
     def _unwrap_predictions_response(
         self,
-        response_json: dict[str, Any],
+        response_json: _PredictionsEnvelope,
     ) -> dict[str, object]:
         """
         Unwrap the Vertex Gemma predictions format to OpenAI format.
@@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
                 message=f"Request failed: {response.text}",
             )
 
-        response_json: Final = response.json()
+        response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
 
         # Unwrap predictions to get OpenAI-compatible response
         openai_response: Final = self._unwrap_predictions_response(response_json)
@@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
                 message=f"Request failed: {response.text}",
             )
 
-        response_json: Final = response.json()
+        response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
 
         # Unwrap predictions to get OpenAI-compatible response
         openai_response: Final = self._unwrap_predictions_response(response_json)

litellm/llms/volcengine/responses/transformation.py

@@ -3,6 +3,7 @@ from types import UnionType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 from pydantic import fields as pyd_fields
 
 import litellm
@@ -37,6 +38,8 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
 
 class _EventModelClass(Protocol):
     @property
@@ -420,7 +423,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
     @staticmethod
     def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]:
-        return raw_response.json()
+        return _RESPONSE_BODY.validate_python(raw_response.json())
 
     @staticmethod
     def _annotation_origin(annotation: object) -> object:

litellm/model_prices_and_context_window_backup.json

@@ -21589,7 +21589,13 @@
         ],
         "supports_vision": true,
         "supports_audio_input": true,
-        "supports_video_input": true
+        "supports_video_input": true,
+        "input_cost_per_token_priority": 5.6252e-07,
+        "output_cost_per_token_priority": 3.37498e-06,
+        "cache_read_input_token_cost_priority": 5.628e-08,
+        "cache_creation_input_token_cost_priority": 5.6252e-07,
+        "input_cost_per_audio_token_priority": 1.12497e-06,
+        "cache_read_input_audio_token_cost_priority": 1.1249e-07
     },
     "databricks/databricks-gemini-3-1-flash-image": {
         "cache_creation_input_token_cost": 6.2503e-07,
@@ -21676,7 +21682,14 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_video_input": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.50002e-06,
+        "output_cost_per_token_priority": 2.699998e-05,
+        "cache_read_input_token_cost_priority": 4.5003e-07,
+        "cache_creation_input_token_cost_priority": 4.50002e-06,
+        "input_cost_per_token_above_200k_tokens_priority": 8.99997e-06,
+        "output_cost_per_token_above_200k_tokens_priority": 4.049997e-05,
+        "cache_read_input_token_cost_above_200k_tokens_priority": 8.9999e-07
     },
     "databricks/databricks-gemini-3-flash": {
         "cache_creation_input_token_cost": 6.2503e-07,
@@ -21710,7 +21723,13 @@
         ],
         "supports_vision": true,
         "supports_audio_input": true,
-        "supports_video_input": true
+        "supports_video_input": true,
+        "input_cost_per_token_priority": 1.12497e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.1249e-07,
+        "cache_creation_input_token_cost_priority": 1.12497e-06,
+        "input_cost_per_audio_token_priority": 2.25001e-06,
+        "cache_read_input_audio_token_cost_priority": 2.2498e-07
     },
     "databricks/databricks-gemini-3-pro": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -21757,7 +21776,11 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-7-flash": {
         "cache_creation_input_token_cost": 7.4998e-07,
@@ -21785,10 +21808,14 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-6-flash": {
-        "cache_creation_input_token_cost": 1.87502e-06,
+        "cache_creation_input_token_cost": 7.4998e-07,
         "cache_read_input_token_cost": 7.497e-08,
         "input_cost_per_token": 7.4998e-07,
         "input_dbu_cost_per_token": 1.0714e-05,
@@ -21816,7 +21843,11 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-5-flash": {
         "cache_creation_input_token_cost": 1.87502e-06,
@@ -21846,7 +21877,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 3.37498e-06,
+        "output_cost_per_token_priority": 2.025002e-05,
+        "cache_read_input_token_cost_priority": 3.3747e-07,
+        "cache_creation_input_token_cost_priority": 3.37498e-06
     },
     "databricks/databricks-gemini-3-5-flash-lite": {
         "cache_creation_input_token_cost": 3.7499e-07,
@@ -21878,7 +21913,11 @@
         "supports_response_schema": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 6.7501e-07,
+        "output_cost_per_token_priority": 5.62499e-06,
+        "cache_read_input_token_cost_priority": 6.748e-08,
+        "cache_creation_input_token_cost_priority": 6.7501e-07
     },
     "databricks/databricks-gemma-3-12b": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -21921,7 +21960,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": false,
+        "input_cost_per_token_priority": 2.45e-06,
+        "output_cost_per_token_priority": 7.7e-06,
+        "cache_read_input_token_cost_priority": 4.55e-07,
+        "cache_creation_input_token_cost_priority": 2.45e-06
     },
     "databricks/databricks-glm-5-3": {
         "cache_creation_input_token_cost": 1.4e-06,
@@ -21951,7 +21994,11 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": false,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 2.45e-06,
+        "output_cost_per_token_priority": 7.7e-06,
+        "cache_read_input_token_cost_priority": 4.55e-07,
+        "cache_creation_input_token_cost_priority": 2.45e-06
     },
     "databricks/databricks-glm-5-3-flash": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -21980,7 +22027,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 2.625e-07,
+        "output_cost_per_token_priority": 8.75e-07,
+        "cache_read_input_token_cost_priority": 5.25e-08,
+        "cache_creation_input_token_cost_priority": 2.625e-07
     },
     "databricks/databricks-gpt-5": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -21999,7 +22050,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 2.49998e-06,
+        "output_cost_per_token_priority": 1.999998e-05,
+        "cache_read_input_token_cost_priority": 2.4997e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06
     },
     "databricks/databricks-gpt-5-1": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -22018,7 +22073,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 2.49998e-06,
+        "output_cost_per_token_priority": 1.999998e-05,
+        "cache_read_input_token_cost_priority": 2.4997e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06
     },
     "databricks/databricks-gpt-5-2": {
         "cache_creation_input_token_cost": 1.75e-06,
@@ -22037,7 +22096,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 3.5e-06,
+        "output_cost_per_token_priority": 2.8e-05,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "cache_creation_input_token_cost_priority": 3.5e-06
     },
     "databricks/databricks-gpt-5-3-codex": {
         "cache_creation_input_token_cost": 1.75e-06,
@@ -22056,7 +22119,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 3.5e-06,
+        "output_cost_per_token_priority": 2.8e-05,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "cache_creation_input_token_cost_priority": 3.5e-06
     },
     "databricks/databricks-gpt-5-4": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -22087,7 +22154,14 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.00003e-06,
+        "output_cost_per_token_priority": 2.999997e-05,
+        "cache_read_input_token_cost_priority": 5.0001e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 5.00003e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.999997e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 5.0001e-07
     },
     "databricks/databricks-gpt-5-4-mini": {
         "cache_creation_input_token_cost": 7.4998e-07,
@@ -22115,7 +22189,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.50003e-06,
+        "output_cost_per_token_priority": 8.99997e-06,
+        "cache_read_input_token_cost_priority": 1.5001e-07,
+        "cache_creation_input_token_cost_priority": 1.50003e-06
     },
     "databricks/databricks-gpt-5-4-nano": {
         "cache_creation_input_token_cost": 1.9999e-07,
@@ -22175,7 +22253,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 8.00002e-06,
+        "output_cost_per_token_priority": 4.000003e-05,
+        "cache_read_input_token_cost_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_priority": 9.99999e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 1.599997e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 6.000001e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.59999e-06,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 1.999998e-05
     },
     "databricks/databricks-gpt-5-6-terra": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -22207,7 +22293,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 2.399999e-05,
+        "cache_read_input_token_cost_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 3.600002e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-5-6-luna": {
         "cache_creation_input_token_cost": 2.4997e-07,
@@ -22239,7 +22333,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 3.9998e-07,
+        "output_cost_per_token_priority": 2.40002e-06,
+        "cache_read_input_token_cost_priority": 3.997e-08,
+        "cache_creation_input_token_cost_priority": 5.0001e-07,
+        "input_cost_per_token_above_272k_tokens_priority": 8.0003e-07,
+        "output_cost_per_token_above_272k_tokens_priority": 3.60003e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.001e-08,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 1.00002e-06
     },
     "databricks/databricks-gpt-5-5": {
         "cache_creation_input_token_cost": 5.00003e-06,
@@ -22270,7 +22372,14 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.249997e-05,
+        "output_cost_per_token_priority": 7.50001e-05,
+        "cache_read_input_token_cost_priority": 1.24999e-06,
+        "cache_creation_input_token_cost_priority": 1.249997e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 1.249997e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 7.50001e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.24999e-06
     },
     "databricks/databricks-gpt-5-5-pro": {
         "cache_creation_input_token_cost": 2.999997e-05,
@@ -22319,7 +22428,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 4.5003e-07,
+        "output_cost_per_token_priority": 3.60003e-06,
+        "cache_read_input_token_cost_priority": 4.501e-08,
+        "cache_creation_input_token_cost_priority": 4.5003e-07
     },
     "databricks/databricks-gpt-5-nano": {
         "cache_creation_input_token_cost": 4.998e-08,
@@ -22469,7 +22582,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.25e-06,
+        "output_cost_per_token_priority": 2.625e-05,
+        "cache_read_input_token_cost_priority": 5.25e-07,
+        "cache_creation_input_token_cost_priority": 5.25e-06
     },
     "databricks/system.ai.kimi-k3": {
         "litellm_provider": "databricks",
@@ -22551,7 +22668,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": false,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 3.85e-07,
+        "output_cost_per_token_priority": 3.85e-06,
+        "cache_read_input_token_cost_priority": 3.85e-07,
+        "cache_creation_input_token_cost_priority": 3.85e-07
     },
     "databricks/databricks-qwen3-next-80b-a3b-instruct": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -61865,6 +61986,39 @@
         "supports_reasoning": true,
         "supports_vision": false
     },
+    "tencent/seedream-image-v5.0-flash": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.018,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://cloud.tencent.com/document/product/1823/130055",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
+    "tencent/seedream-image-v5.0-lite": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.035,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://www.tencentcloud.com/document/product/1300/78937",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
+    "tencent/seedream-image-v5.0-pro": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.09,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://www.tencentcloud.com/document/product/1300/78937",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
     "cognition/swe-1.6": {
         "input_cost_per_token": 5e-07,
         "output_cost_per_token": 2.5e-06,
@@ -77774,7 +77928,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.25e-07,
+        "output_cost_per_token_priority": 2.1e-06,
+        "cache_read_input_token_cost_priority": 5.25e-08,
+        "cache_creation_input_token_cost_priority": 5.25e-07
     },
     "databricks/databricks-deepseek-v4-1-flash-1017": {
         "cache_creation_input_token_cost": 3.0002e-07,
@@ -77897,7 +78055,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_video_input": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.50002e-06,
+        "output_cost_per_token_priority": 2.250003e-05,
+        "cache_read_input_token_cost_priority": 2.2498e-07,
+        "cache_creation_input_token_cost_priority": 4.50002e-06
     },
     "databricks/databricks-gpt-5-4-pro": {
         "cache_creation_input_token_cost": 2.999997e-05,
@@ -77959,7 +78121,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 1.99997e-05,
+        "cache_read_input_token_cost_priority": 1.9999e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-6-1-terra": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -77991,7 +78161,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.99997e-06,
+        "output_cost_per_token_priority": 9.99999e-06,
+        "cache_read_input_token_cost_priority": 1.0003e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 4.00001e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 1.500002e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06
     },
     "databricks/databricks-gpt-6-astra": {
         "cache_creation_input_token_cost": 1.249997e-05,
@@ -78023,7 +78201,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.999998e-05,
+        "output_cost_per_token_priority": 9.99999e-05,
+        "cache_read_input_token_cost_priority": 1.99997e-06,
+        "cache_creation_input_token_cost_priority": 2.500001e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 4.000003e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 0.0001500002,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05
     },
     "databricks/databricks-gpt-6-luna": {
         "cache_creation_input_token_cost": 1.2502e-07,
@@ -78055,7 +78241,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.9999e-07,
+        "output_cost_per_token_priority": 1.0003e-06,
+        "cache_read_input_token_cost_priority": 2.002e-08,
+        "cache_creation_input_token_cost_priority": 2.4997e-07,
+        "input_cost_per_token_above_272k_tokens_priority": 3.9998e-07,
+        "output_cost_per_token_above_272k_tokens_priority": 1.5001e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07
     },
     "databricks/databricks-gpt-6-sol": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -78087,7 +78281,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 1.99997e-05,
+        "cache_read_input_token_cost_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-image-1": {
         "cache_creation_input_token_cost": 5.00003e-06,

litellm/provider_endpoints_support_backup.json

@@ -3793,8 +3793,8 @@
         "responses": true,
         "text_completion": true,
         "embeddings": false,
-        "image_generations": false,
-        "image_edits": false,
+        "image_generations": true,
+        "image_edits": true,
         "audio_transcriptions": false,
         "audio_speech": false,
         "rerank": false,

litellm/proxy/_lazy_openapi_snapshot.json

@@ -3487,6 +3487,13 @@
               "title": "Entities",
               "type": "object"
             },
+            "guardrails": {
+              "additionalProperties": {
+                "$ref": "#/components/schemas/GuardrailMetric"
+              },
+              "title": "Guardrails",
+              "type": "object"
+            },
             "mcp_servers": {
               "additionalProperties": {
                 "$ref": "#/components/schemas/MetricWithMetadata"
@@ -3802,6 +3809,74 @@
           "title": "ExportType",
           "type": "string"
         },
+        "GuardrailKeyMetric": {
+          "properties": {
+            "metadata": {
+              "$ref": "#/components/schemas/KeyMetadata"
+            },
+            "metrics": {
+              "$ref": "#/components/schemas/GuardrailSpendMetrics"
+            }
+          },
+          "required": [
+            "metrics",
+            "metadata"
+          ],
+          "title": "GuardrailKeyMetric",
+          "type": "object"
+        },
+        "GuardrailMetadata": {
+          "properties": {
+            "guardrail_name": {
+              "title": "Guardrail Name",
+              "type": "string"
+            }
+          },
+          "required": [
+            "guardrail_name"
+          ],
+          "title": "GuardrailMetadata",
+          "type": "object"
+        },
+        "GuardrailMetric": {
+          "properties": {
+            "api_key_breakdown": {
+              "additionalProperties": {
+                "$ref": "#/components/schemas/GuardrailKeyMetric"
+              },
+              "title": "Api Key Breakdown",
+              "type": "object"
+            },
+            "metadata": {
+              "$ref": "#/components/schemas/GuardrailMetadata"
+            },
+            "metrics": {
+              "$ref": "#/components/schemas/GuardrailSpendMetrics"
+            }
+          },
+          "required": [
+            "metrics",
+            "metadata"
+          ],
+          "title": "GuardrailMetric",
+          "type": "object"
+        },
+        "GuardrailSpendMetrics": {
+          "properties": {
+            "spend": {
+              "default": 0.0,
+              "title": "Spend",
+              "type": "number"
+            },
+            "untracked_units": {
+              "default": 0,
+              "title": "Untracked Units",
+              "type": "integer"
+            }
+          },
+          "title": "GuardrailSpendMetrics",
+          "type": "object"
+        },
         "HTTPAuthSecurityScheme": {
           "description": "Defines a security scheme using HTTP authentication.",
           "properties": {

litellm/proxy/auth/user_api_key_auth.py

@@ -370,7 +370,7 @@ class _KeyModelBudgetLimiter(Protocol):
 
 class _UserModelBudgetLimiter(Protocol):
     async def is_user_within_model_budget(
-        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str
+        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str, team_id: str | None = None
     ) -> bool: ...
 
 
@@ -388,11 +388,15 @@ class _EveryModelBudgetLimiter(Protocol):
     async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
 
     async def is_user_within_model_budget(
-        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str
+        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str, team_id: str | None = None
     ) -> bool: ...
 
     async def is_end_user_within_model_budget(
-        self, end_user_id: str, end_user_model_max_budget: Mapping[str, object], model: str
+        self,
+        end_user_id: str,
+        end_user_model_max_budget: Mapping[str, object],
+        model: str,
+        team_id: str | None = None,
     ) -> bool: ...
 
     async def is_team_within_model_budget(
@@ -503,6 +507,7 @@ async def _check_user_model_budget(
             user_id=valid_token.user_id,
             user_model_max_budget=user_model_max_budget,
             model=model_name,
+            team_id=valid_token.team_id,
         )
 
 
@@ -547,6 +552,7 @@ async def _is_within_every_model_budget(
                 user_id=valid_token.user_id,
                 user_model_max_budget=user_model_max_budget,
                 model=model,
+                team_id=valid_token.team_id,
             )
         if (
             valid_token.end_user_id is not None
@@ -557,6 +563,7 @@ async def _is_within_every_model_budget(
                 end_user_id=valid_token.end_user_id,
                 end_user_model_max_budget=end_user_model_max_budget,
                 model=model,
+                team_id=valid_token.team_id,
             )
         if valid_token.team_id is not None and isinstance(team_model_max_budget, Mapping) and team_model_max_budget:
             await model_max_budget_limiter.is_team_within_model_budget(
@@ -3154,6 +3161,7 @@ async def validate_resolved_virtual_key(  # noqa: C901  # Preserve ordering of e
                             end_user_id=valid_token.end_user_id,
                             end_user_model_max_budget=end_user_mmb,
                             model=model_name,
+                            team_id=valid_token.team_id,
                         )
 
         # Check 6: Additional Common Checks across jwt + key auth
@@ -4733,6 +4741,7 @@ async def _run_post_custom_auth_checks(
                 end_user_id=valid_token.end_user_id,
                 end_user_model_max_budget=end_user_mmb,
                 model=model_name,
+                team_id=valid_token.team_id,
             )
 
     # team / user / end_user / project context objects are fetched by

litellm/proxy/client/README.md

@@ -342,7 +342,7 @@ The CLI provides these authentication commands:
 - **`lite login --pkce`** - Sign in through the system browser with OAuth authorization code + PKCE; the key renews itself with a refresh token
 - **`lite logout`** - Clear stored authentication token (and revoke a `--pkce` refresh token on the proxy)
 - **`lite whoami`** - Show current authentication status
-- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); renews a `--pkce` key first and fails once a classic token has expired
+- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); renews a `--pkce` key first, and fails once a classic token is past 24 hours and the proxy rejects it
 
 ### Authentication Flow Steps
 
@@ -382,7 +382,7 @@ Keychain storage needs the `keyring` package, which ships with `pip install 'lit
 
 `lite logout` clears both stores. If the keychain is locked at that moment it says so, and re-running it once the keychain is unlocked finishes the job.
 
-The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while fresh and fails once it expires -- there is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. A credential from `lite login --pkce` is the exception: it carries a refresh token, so the CLI renews the key shortly before it expires and `lite logout` revokes the refresh token on the proxy (see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
+The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it, and once it is past 24 hours (or the client's `LITELLM_CLI_JWT_EXPIRATION_HOURS`) asks the proxy first and fails if the proxy rejects it. There is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. A credential from `lite login --pkce` is the exception: it carries a refresh token, so the CLI renews the key shortly before it expires and `lite logout` revokes the refresh token on the proxy (see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
 
 ### Usage
 

litellm/proxy/client/cli/README.md

@@ -506,7 +506,7 @@ To pin the model, pass the agent's own model flag (for example `lite claude --mo
 
 The token minted by `lite login` is a short-lived, per-session agent credential, not a managed virtual key. It is scoped to the user and team you authenticated as, inherits that user's and team's models and budgets, and is enforced on the proxy exactly like a virtual key on the same team (guardrails, routing, logging, spend). Spend is tracked against the shared team and user budgets, so running several agents (or logging in more than once) does not hand each session its own separate budget; they all draw down the same team/user allowance. There is no separate per-session cap, so sustained agent use is not capped at a small chat-session limit.
 
-The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while it's still fresh and fails once it expires -- there is no silent renewal, so a long-running session needs a fresh `lite login` once a day. `lite claude`, `lite codex`, `lite opencode`, and `lite pi` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. `lite login --pkce` is the exception to the daily re-login: it signs in through your system browser with OAuth authorization code and PKCE and stores a refresh token next to the key, so every `lite` command and `lite auth print-token` renew the key on their own shortly before it expires, `lite whoami` shows when the current key expires, and `lite logout` revokes the refresh token on the proxy (it needs a proxy that serves `/.well-known/litellm-cli-auth`; see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). When a renewal is refused, for example after a `lite logout` run from another copy of the credential, the command prints why on stderr and, once the key has run out, tells you to run `lite login --pkce` again. Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
+The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it without asking the proxy for its first 24 hours, or `LITELLM_CLI_JWT_EXPIRATION_HOURS` if you set it on the client. After that, `print-token` and `lite whoami` ask the proxy whether it still accepts the token, and `print-token` fails once the proxy rejects it. There is no silent renewal, so a long-running session needs a fresh `lite login` each time the proxy's token lifetime runs out. If the proxy's tokens last less than 24 hours, set the same `LITELLM_CLI_JWT_EXPIRATION_HOURS` on the client so `print-token` stops printing the token once it runs out. `lite up` still starts a new login once the token is older than that same client window. It copies the token into Claude Code's settings as a static key that it never renews, so a fresh login gives Claude Code the proxy's full token lifetime instead of whatever an old token has left. `lite claude`, `lite codex`, `lite opencode`, and `lite pi` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. `lite login --pkce` is the exception to the re-login: it signs in through your system browser with OAuth authorization code and PKCE and stores a refresh token next to the key, so every `lite` command and `lite auth print-token` renew the key on their own shortly before it expires, `lite whoami` shows when the current key expires, and `lite logout` revokes the refresh token on the proxy (it needs a proxy that serves `/.well-known/litellm-cli-auth`; see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). When a renewal is refused, for example after a `lite logout` run from another copy of the credential, the command prints why on stderr and, once the key has run out, tells you to run `lite login --pkce` again. Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
 
 ### Route Every Claude Code Session Through the Proxy
 

litellm/proxy/client/cli/commands/auth.py

@@ -13,7 +13,6 @@ from rich.console import Console
 from rich.table import Table
 from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never
 
-from litellm.constants import CLI_JWT_EXPIRATION_HOURS
 from litellm.litellm_core_utils.cli_keyring import (
     DISABLE_KEYRING_ENV_VAR,
     SYSTEM_KEYRING,
@@ -143,6 +142,9 @@ _TeamMapping: Final = TypeVar("_TeamMapping", bound=Mapping[str, object])
 
 _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
 
+_PROXY_KEY_CHECK_TIMEOUT_SECONDS: Final = 3
+_PROXY_REJECTED_KEY_STATUSES: Final = frozenset({401, 403})
+
 KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'"
 
 KEYRING_ENABLE_HINT: Final = "keyring --enable (or unset PYTHON_KEYRING_BACKEND)"
@@ -1033,10 +1035,12 @@ def print_token(ctx: click.Context):
 
     Designed to be used as Claude Code's `apiKeyHelper`
     (https://docs.claude.com/en/docs/claude-code/settings): stdout must
-    contain only the token, so all diagnostics go to stderr. The token
-    expires after `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h); a
-    `lite login --pkce` token renews itself here first, and once a token
-    has expired for good, run the same `lite login` command again.
+    contain only the token, so all diagnostics go to stderr. A token younger
+    than `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h) is printed without
+    asking the proxy. An older one is checked against the proxy and refused
+    only if the proxy rejects it. A `lite login --pkce`
+    token renews itself here first, and once a token has expired for good,
+    run the same `lite login` command again.
     """
     vault: Final = context_secret_vault(ctx)
     token_data: Final = load_token(vault=vault)
@@ -1055,7 +1059,7 @@ def print_token(ctx: click.Context):
         sys.exit(1)
 
     renews: Final = "refresh_token" in token_data
-    if not is_cli_token_fresh(token_data) and not renews:
+    if not renews and token_expired(token_data):
         click.echo("Token expired. Run 'lite login' again.", err=True)
         sys.exit(1)
 
@@ -1111,8 +1115,32 @@ def whoami(ctx: click.Context):
     expires_at: Final = token_data.get("expires_at")
     if isinstance(expires_at, (int, float)):
         click.echo(_key_expiry_line(expires_at, renews="refresh_token" in token_data))
-    elif age_hours > CLI_JWT_EXPIRATION_HOURS:
-        click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
+    elif token_expired(token_data):
+        click.echo("Token expired. Run 'lite login' again.")
+
+
+def token_expired(token_data: Mapping[str, object]) -> bool:
+    if is_cli_token_fresh(token_data):
+        return False
+    if isinstance(token_data.get("expires_at"), (int, float)):
+        return True
+    return _proxy_rejects_key(token_data)
+
+
+def _proxy_rejects_key(token_data: Mapping[str, object]) -> bool:
+    key: Final = token_data.get("key")
+    base_url: Final = token_data.get("base_url")
+    if not isinstance(key, str) or not isinstance(base_url, str) or not base_url:
+        return True
+    try:
+        response: Final = requests.get(
+            f"{base_url}/v1/models",
+            headers={"Authorization": f"Bearer {key}"},
+            timeout=_PROXY_KEY_CHECK_TIMEOUT_SECONDS,
+        )
+    except requests.RequestException:
+        return False
+    return response.status_code in _PROXY_REJECTED_KEY_STATUSES
 
 
 def _key_expiry_line(expires_at: float, renews: bool) -> str:

litellm/proxy/client/models.py

@@ -3,9 +3,12 @@ from collections.abc import Mapping
 from typing import Any, Final
 
 import requests
+from pydantic import ConfigDict, TypeAdapter
 
 from .exceptions import NotFoundError, UnauthorizedError
 
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
 
 class ModelsManagementClient:
     def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
@@ -109,7 +112,7 @@ class ModelsManagementClient:
         try:
             response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
-            return response.json()
+            return _JSON_OBJECT.validate_python(response.json())
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 401:
                 raise UnauthorizedError(e)
@@ -145,7 +148,7 @@ class ModelsManagementClient:
         try:
             response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
-            return response.json()
+            return _JSON_OBJECT.validate_python(response.json())
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 401:
                 raise UnauthorizedError(e)
@@ -287,7 +290,7 @@ class ModelsManagementClient:
         try:
             response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
-            return response.json()
+            return _JSON_OBJECT.validate_python(response.json())
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 401:
                 raise UnauthorizedError(e)

litellm/proxy/common_request_processing.py

@@ -4,6 +4,7 @@ import json
 import logging
 import math
 from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence
+from dataclasses import dataclass
 from datetime import datetime
 from functools import lru_cache
 from itertools import chain
@@ -136,7 +137,7 @@ from litellm.proxy.utils import (  # noqa: F401  # legacy module exports
     _check_and_merge_model_level_guardrails,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
     check_and_merge_model_level_guardrails,
 )
-from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
+from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
 from litellm.router import Router
 from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
 from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
@@ -265,6 +266,8 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
     AnthropicStreamModelRestamper,
 )
 from litellm.proxy.litellm_pre_call_utils import (
+    ROUTER_SCHEDULED_ROUTE_TYPES,
+    SCHEDULER_PRIORITY_HEADER,
     LiteLLMProxyRequestSetup,
     add_litellm_data_to_request,
     get_metadata_variable_name,
@@ -2006,6 +2009,49 @@ def _content_text(fields: Mapping[str, object]) -> str:
     return str(fields.get("content", ""))
 
 
+async def _no_stream_failure_to_release() -> None:
+    return None
+
+
+@dataclass(slots=True)
+class _ClientStreamFailureReport:
+    logging_obj: LiteLLMLoggingObj
+    report: Callable[[Exception], Awaitable[None]]
+    held: Exception | None = None
+
+    @classmethod
+    def hold_while_streaming(cls, logging_obj: LiteLLMLoggingObj | None) -> Callable[[], Awaitable[None]]:
+        if logging_obj is None or logging_obj.on_detached_stream_failure is None:
+            return _no_stream_failure_to_release
+        return cls(logging_obj, logging_obj.on_detached_stream_failure)._start_holding()
+
+    def _start_holding(self) -> Callable[[], Awaitable[None]]:
+        self.logging_obj.on_detached_stream_failure = self.hold
+        return self.release
+
+    async def hold(self, failure: Exception) -> None:
+        if self.held is None:
+            self.held = failure
+
+    async def release(self) -> None:
+        self.logging_obj.on_detached_stream_failure = self.report
+        if self.held is None:
+            return
+        await asyncio.shield(
+            spawn_logging_task(
+                asyncio.get_running_loop(), self._report_held(self.held), task_name="stream failure report"
+            )
+        )
+
+    async def _report_held(self, failure: Exception) -> None:
+        try:
+            await self.report(failure)
+        except Exception as hook_failure:  # noqa: BLE001  # a failing failure hook must not skip the stream cleanup
+            verbose_proxy_logger.warning(
+                "Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure)
+            )
+
+
 class ProxyBaseLLMRequestProcessing:
     def __init__(self, data: dict, served_model: str | None = None):
         self.data = data
@@ -2434,14 +2480,20 @@ class ProxyBaseLLMRequestProcessing:
                     self.data["model"] = alias_target
 
         aliased_model: Final = self.data.get("model")
+        if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
+            self.data.update(
+                llm_router.priority_header_update(
+                    model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
+                )
+            )
 
         litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
         self.data["litellm_call_id"] = litellm_call_id
         bind_telemetry_call_id(litellm_call_id)
-        DDSpanTagger.tag_call_id(self.data.get("litellm_call_id"))
+        DDSpanTagger.tag_call_id(litellm_call_id)
         DDSpanTagger.tag_request(
             user_api_key_dict=user_api_key_dict,
-            requested_model=self.data.get("model"),
+            requested_model=aliased_model,
         )
 
         ### AUTO STREAM USAGE TRACKING ###
@@ -3498,6 +3550,8 @@ class ProxyBaseLLMRequestProcessing:
         request_data: Final = self.data
 
         async def _on_detached_stream_failure(exc: Exception) -> None:
+            if not logging_obj.claim_stream_failure_report():
+                return
             try:
                 await proxy_logging_obj.post_call_failure_hook(
                     user_api_key_dict=user_api_key_dict,
@@ -4448,11 +4502,15 @@ class ProxyBaseLLMRequestProcessing:
         client_disconnected = False
         delivered_chunk = False
         recent_tail = SSE_STREAM_START_TAIL  # rebind-ok: rolling window over the yielded bytes
+        stream_logging_obj: Final[object] = request_data.get("litellm_logging_obj")  # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]  # legacy request data is narrowed to Logging below
         guarded_stream: Final[AsyncGenerator[object, None]] = proxy_logging_obj.async_post_call_streaming_iterator_hook(
             user_api_key_dict=user_api_key_dict,
             response=response,
             request_data=request_data,
         )
+        release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(
+            stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
+        )
         try:
             str_so_far = ""
             async for chunk in guarded_stream:
@@ -4523,10 +4581,18 @@ class ProxyBaseLLMRequestProcessing:
             verbose_proxy_logger.exception(
                 "litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e
             )
-            transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
-                user_api_key_dict=user_api_key_dict,
-                original_exception=e,
-                request_data=request_data,
+            already_reported: Final = (
+                isinstance(stream_logging_obj, LiteLLMLoggingObj)
+                and not stream_logging_obj.claim_stream_failure_report()
+            )
+            transformed_exception: Final = (
+                None
+                if already_reported
+                else await proxy_logging_obj.post_call_failure_hook(
+                    user_api_key_dict=user_api_key_dict,
+                    original_exception=e,
+                    request_data=request_data,
+                )
             )
             if transformed_exception is not None:
                 e = transformed_exception
@@ -4549,15 +4615,18 @@ class ProxyBaseLLMRequestProcessing:
             seal: Final = "" if seal_open_frame is None else seal_open_frame(recent_tail)
             yield seal + error_frame if seal else error_frame
         finally:
-            await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
-                request=request,
-                request_data=request_data,
-                response=response,
-                stream_completed=stream_completed,
-                client_disconnected=client_disconnected,
-                user_api_key_dict=user_api_key_dict,
-                proxy_logging_obj=proxy_logging_obj,
-            )
+            try:
+                await release_stream_failure()
+            finally:
+                await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
+                    request=request,
+                    request_data=request_data,
+                    response=response,
+                    stream_completed=stream_completed,
+                    client_disconnected=client_disconnected,
+                    user_api_key_dict=user_api_key_dict,
+                    proxy_logging_obj=proxy_logging_obj,
+                )
 
     @staticmethod
     def async_sse_data_generator(

litellm/proxy/db/accounting_ledger.py

@@ -0,0 +1,269 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import AsyncGenerator, Callable
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias
+
+from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter
+from typing_extensions import assert_never
+
+from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent
+from litellm.proxy.db.db_span import db_span
+from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
+
+if TYPE_CHECKING:
+    from prisma import Prisma
+
+    from litellm.proxy.utils import PrismaClient
+
+AccountingEvent: TypeAlias = InboundUsageEvent | OutboundUsageEvent | CostEvent
+_JSON: Final = TypeAdapter(dict[str, JsonValue])
+_INSTANT: Final[TypeAdapter[datetime]] = TypeAdapter(AwareDatetime)
+_STRINGS: Final = TypeAdapter(tuple[dict[str, str], ...])
+_TIMEOUT: Final = timedelta(seconds=10)
+
+
+class AccountingConflictError(ValueError):
+    pass
+
+
+_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
+
+
+@dataclass(frozen=True, slots=True)
+class _AcceptedEvent:
+    event_id: str
+    kind: EventKind
+    source_event_id: str | None
+    occurred_at: datetime
+    payload: str
+    payload_hash: str
+    targets: tuple[str, ...]
+    schema_version: int
+    version: int
+
+
+def _json(value: object) -> str:
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def _prepare(
+    event: AccountingEvent,
+    targets: tuple[DeliveryTarget, ...],
+    envelope: str,
+    schema_version: int,
+    version: int,
+) -> _AcceptedEvent:
+    if type(schema_version) is not int or type(version) is not int or min(schema_version, version) < 1:
+        raise ValueError("Accounting versions must be positive integers")
+    validated: Final = type(event).model_validate_json(event.model_dump_json())
+    payload: Final = _JSON.validate_json(envelope)
+    if payload.get("event") != validated.model_dump(mode="json"):
+        raise ValueError("Accounting envelope must contain the matching public event")
+    destinations: Final = tuple(sorted(frozenset(DeliveryTarget(target).value for target in targets)))
+    serialized: Final = _json(payload)
+    identity: Final = _json((schema_version, version, destinations, payload))
+    kind: Final = (
+        EventKind.COST
+        if isinstance(validated, CostEvent)
+        else EventKind.INBOUND_USAGE
+        if isinstance(validated, InboundUsageEvent)
+        else EventKind.OUTBOUND_USAGE
+    )
+    return _AcceptedEvent(
+        event_id=validated.event_id,
+        kind=kind,
+        source_event_id=validated.usage_event_id if isinstance(validated, CostEvent) else None,
+        occurred_at=validated.occurred_at if isinstance(validated, CostEvent) else validated.ended_at,
+        payload=serialized,
+        payload_hash=hashlib.sha256(identity.encode()).hexdigest(),
+        targets=destinations,
+        schema_version=schema_version,
+        version=version,
+    )
+
+
+_INSERT_EVENT: Final = """
+INSERT INTO "LiteLLM_AccountingEvent"
+    (event_id, kind, source_event_id, schema_version, payload_hash, occurred_at, payload)
+VALUES ($1, $2::"AccountingEventKind", $3, $4::int, $5, $6::timestamp, $7::jsonb)
+ON CONFLICT (event_id) DO NOTHING RETURNING event_id
+"""
+_INSERT_DELIVERIES: Final = """
+INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
+SELECT $1, value::"AccountingDeliveryTarget", $3::int FROM jsonb_array_elements_text($2::jsonb)
+"""
+_LOCK_SOURCE: Final = """
+SELECT event_id FROM "LiteLLM_AccountingEvent"
+WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE
+"""
+_EXISTING_COST: Final = """
+SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_event_id = $1
+"""
+_LOCK_EVENT: Final = """
+SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
+"""
+_COMPLETED_FAMILY: Final = """
+root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
+AND root.recorded_at < $1::timestamp
+AND (root.kind = 'inbound_usage' OR EXISTS (
+    SELECT 1 FROM "LiteLLM_AccountingEvent" AS cost WHERE cost.source_event_id = root.event_id AND cost.kind = 'cost'
+))
+AND NOT EXISTS (
+    SELECT 1 FROM "LiteLLM_AccountingEvent" AS member
+    WHERE (member.event_id = root.event_id OR (member.kind = 'cost' AND member.source_event_id = root.event_id))
+    AND (member.recorded_at >= $1::timestamp
+        OR EXISTS (SELECT 1 FROM "LiteLLM_AccountingDelivery" AS delivery
+            WHERE delivery.event_id = member.event_id AND delivery.state != 'applied'))
+)
+"""
+_RETENTION_CANDIDATES: Final = (
+    'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE '
+    + _COMPLETED_FAMILY
+    + " ORDER BY root.recorded_at, root.event_id LIMIT $2::int FOR UPDATE OF root SKIP LOCKED"
+)
+_ELIGIBLE_GROUP: Final = (
+    'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE root.event_id = $2 AND ' + _COMPLETED_FAMILY
+)
+_DELETE_DELIVERIES: Final = """
+DELETE FROM "LiteLLM_AccountingDelivery" WHERE event_id IN (
+    SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
+)
+"""
+_DELETE_EVENTS: Final = """
+DELETE FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
+"""
+
+
+_Conflict: TypeAlias = Literal["missing_source", "competing_cost", "identity_conflict"]
+
+
+async def _accept(db: Prisma, record: _AcceptedEvent) -> Literal["inserted", "duplicate"] | _Conflict:
+    if record.source_event_id is not None:
+        source: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_SOURCE, record.source_event_id))
+        if not source:
+            return "missing_source"
+        costs: Final = _STRINGS.validate_python(tuple(await db.query_raw(_EXISTING_COST, record.source_event_id)))
+        if any(cost["event_id"] != record.event_id for cost in costs):
+            return "competing_cost"
+    inserted: Final = _STRINGS.validate_python(
+        await db.query_raw(
+            _INSERT_EVENT,
+            record.event_id,
+            record.kind.value,
+            record.source_event_id,
+            record.schema_version,
+            record.payload_hash,
+            record.occurred_at.astimezone(timezone.utc).isoformat(),
+            record.payload,
+        )
+    )
+    if not inserted:
+        existing: Final = _STRINGS.validate_python(tuple(await db.query_raw(_LOCK_EVENT, record.event_id)))
+        if not existing or existing[0]["payload_hash"] != record.payload_hash:
+            return "identity_conflict"
+        return "duplicate"
+    await db.execute_raw(_INSERT_DELIVERIES, record.event_id, _json(record.targets), record.version)
+    return "inserted"
+
+
+async def _delete_completed(db: Prisma, event_id: str, cutoff: str) -> int:
+    if not _STRINGS.validate_python(await db.query_raw(_ELIGIBLE_GROUP, cutoff, event_id)):
+        return 0
+    await db.execute_raw(_DELETE_DELIVERIES, event_id)
+    return await db.execute_raw(_DELETE_EVENTS, event_id)
+
+
+class AccountingLedger:
+    def __init__(self, transaction: Callable[[], _Transaction]) -> None:
+        self._transaction: Final = transaction
+
+    @classmethod
+    def for_client(cls, client: PrismaClient) -> AccountingLedger:
+        from prisma.client import TransactionManager
+
+        managers: Final = TypeAdapter(TransactionManager, config=ConfigDict(arbitrary_types_allowed=True))
+
+        @asynccontextmanager
+        async def transaction() -> AsyncGenerator[Prisma]:
+            primary: Final = writer_wrapper(client.db)
+            manager: Final = managers.validate_python(primary.tx(timeout=_TIMEOUT))
+            async with db_span("accounting_ledger", "LiteLLM_AccountingEvent"), manager as db:
+                await db.execute_raw("SET LOCAL statement_timeout = 5000")
+                await db.execute_raw("SET LOCAL lock_timeout = 1000")
+                yield db
+
+        return cls(transaction)
+
+    async def insert_event(
+        self,
+        event: AccountingEvent,
+        targets: tuple[DeliveryTarget, ...],
+        *,
+        envelope: str,
+        schema_version: int = 1,
+        version: int = 1,
+    ) -> Literal["inserted", "duplicate"]:
+        """Accept T5/T6's serialized envelope, containing its public event under `event`.
+
+        The immutable acceptance hash includes the target set and delivery version.
+        Changing either for an accepted event conflicts. T7 must provide a separate
+        add-target or new-version workflow that preserves receipts and prevents
+        duplicate contributions; redelivery cannot silently create or reset work.
+
+        The Prisma adapter opens one interactive transaction per event, including
+        two SET LOCAL statements and a source-row lock for costs. I/O is async,
+        but awaiting this method inline adds commit latency per event, potentially
+        several times per request. T5 owns bounded asynchronous capture and an
+        explicit batching strategy; looping over this method does not batch commits.
+        Success acknowledges commit, independently of HTTP response acknowledgement.
+        """
+        record: Final = _prepare(event, targets, envelope, schema_version, version)
+        async with self._transaction() as db:
+            result: Final = await _accept(db, record)
+        match result:
+            case "inserted" | "duplicate":
+                return result
+            case "missing_source":
+                raise AccountingConflictError("Cost source must be a durably accepted outbound usage event")
+            case "competing_cost":
+                raise AccountingConflictError("Usage event already has an authoritative cost")
+            case "identity_conflict":
+                raise AccountingConflictError("Event identity conflicts with its accepted content or delivery set")
+            case _:
+                assert_never(result)
+
+    async def purge_completed(
+        self,
+        *,
+        redelivery_before: datetime | None = None,
+        recovery_before: datetime | None = None,
+        limit: int = 100,
+    ) -> int:
+        """Delete at most `limit` completed families once both retention owners permit it.
+
+        T8 must supply a coherent recovery watermark including outstanding holds.
+        Omitting either watermark disables deletion. Each usage row and its cost
+        remain together so a retained charge never loses its source or vice versa.
+        Applied receipts are deleted with the family, ending its retained dedup
+        protection. The caller must guarantee events recorded before
+        redelivery_before cannot be redelivered through supported paths.
+        T8's recovery_before must also protect checkpoints,
+        outstanding holds and unfinished settlements; age alone is insufficient.
+        """
+        if type(limit) is not int or not 1 <= limit <= 1000:
+            raise ValueError("Accounting retention batch size must be between 1 and 1000")
+        if redelivery_before is None or recovery_before is None:
+            return 0
+        cutoff: Final = min(
+            _INSTANT.validate_python(redelivery_before).astimezone(timezone.utc),
+            _INSTANT.validate_python(recovery_before).astimezone(timezone.utc),
+        ).isoformat()
+        async with self._transaction() as db:
+            roots: Final = _STRINGS.validate_python(tuple(await db.query_raw(_RETENTION_CANDIDATES, cutoff, limit)))
+            deleted: Final = tuple([await _delete_completed(db, root["event_id"], cutoff) for root in roots])
+        return sum(deleted)

litellm/proxy/db/budget_window_spend_writer.py

@@ -64,6 +64,16 @@ _ROLL_WINDOW_SPEND_SQL: Final = (
     "AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')"
 )
 
+_OPEN_WINDOW_SPEND_SQL: Final = (
+    'INSERT INTO "LiteLLM_BudgetWindowSpend" '
+    "(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) "
+    "VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), 0, "
+    "($4::timestamptz AT TIME ZONE 'UTC'), ($4::timestamptz AT TIME ZONE 'UTC')) "
+    "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET "
+    "window_start = EXCLUDED.window_start, spend = 0, updated_at = EXCLUDED.updated_at "
+    "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
+)
+
 _SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
     "SELECT COALESCE(SUM(spend), 0.0) AS total, "
     "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
@@ -389,3 +399,29 @@ async def roll_window_spend_row(
             to_naive_utc(new_window_start),
             to_naive_utc(datetime.now(timezone.utc)),
         )
+
+
+async def open_window_spend_row(
+    prisma_client: "PrismaClient",
+    entity_type: str,
+    entity_id: str,
+    window_duration: str,
+    window_start: datetime,
+    replaces_window_start: datetime | None,
+) -> None:
+    """Start a window at `window_start` unless another writer already started one.
+
+    The row is created when it does not exist, and moved onto the new window only
+    while it still holds `replaces_window_start`, the ended window the caller saw.
+    A writer that lost the race leaves the winner's window in place, so reading
+    the row back gives every writer the same window start.
+    """
+    async with db_span("open_window_spend_row", "LiteLLM_BudgetWindowSpend"):
+        await prisma_client.db.execute_raw(
+            _OPEN_WINDOW_SPEND_SQL,
+            entity_type,
+            entity_id,
+            window_duration,
+            to_naive_utc(window_start),
+            None if replaces_window_start is None else to_naive_utc(replaces_window_start),
+        )

litellm/proxy/guardrails/_content_utils.py

@@ -305,6 +305,11 @@ def iter_message_text(data: Mapping[str, object]) -> Iterator[str]:
         yield from _iter_text_parts_in_content(message.get("content"))
 
 
+def _is_tool_message(message: object) -> bool:
+    fields: Final = _text_object(message)
+    return fields is not None and str(fields.get("role") or "").lower() == "tool"
+
+
 def _call_item(item: object) -> tuple[str, object] | None:
     """Return ``(field, value)`` for a Responses tool-call input item, else None."""
     fields: Final = _text_object(item)
@@ -316,11 +321,14 @@ def walk_user_text(
     data: dict[str, object],
     visit: Callable[[str], str],
     visit_tool: Callable[[JsonValue], JsonValue] | None = None,
+    *,
+    skip_tool_message: bool = False,
 ) -> int:
     """Rewrite every text fragment in place via ``visit``.
 
     Mutates ``data["messages"]`` and ``data["input"]`` (including tool payloads), ``data["system"]`` and
-    ``data["instructions"]``. Returns the number
+    ``data["instructions"]``. ``skip_tool_message`` leaves ``role: tool`` messages and ``tool_result``
+    blocks as sent. Returns the number
     of fragments visited so callers can short-circuit when nothing was
     inspected.
     """
@@ -353,6 +361,8 @@ def walk_user_text(
 
     def _rewrite_part(part: Mapping[str, object], native: bool) -> Mapping[str, object]:
         nonlocal visited
+        if skip_tool_message and part.get("type") == "tool_result":
+            return part
         text: Final = _part_text(part)
         rewritten: Final = {**part, **({"text": visit(text)} if text is not None else {})}
         if text is not None:
@@ -384,18 +394,17 @@ def walk_user_text(
         return content
 
     messages: Final = data.get("messages")
-    if isinstance(messages, list):
-        for message in messages:
-            if not isinstance(message, dict):
-                continue
-            if "content" in message:
-                message["content"] = _rewrite_content(message["content"])
-            calls = message.get("tool_calls")
-            if isinstance(calls, list):
-                for call in calls:
-                    function = call.get("function") if isinstance(call, dict) else None
-                    if isinstance(function, dict) and "arguments" in function:
-                        function["arguments"] = _rewrite_arguments(function["arguments"])
+    for message in _object_list(messages) or ():
+        if (skip_tool_message and _is_tool_message(message)) or not isinstance(message, dict):
+            continue
+        if "content" in message:
+            message["content"] = _rewrite_content(message["content"])
+        calls = message.get("tool_calls")
+        if isinstance(calls, list):
+            for call in calls:
+                function = call.get("function") if isinstance(call, dict) else None
+                if isinstance(function, dict) and "arguments" in function:
+                    function["arguments"] = _rewrite_arguments(function["arguments"])
 
     if "system" in data:
         data["system"] = _rewrite_content(data["system"])  # rebind-ok: preserve the native hook's in-place contract

litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py

@@ -169,7 +169,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
 
         from litellm.types.llms.openai import OpenAIModerationResponse
 
-        return OpenAIModerationResponse(**response.json())
+        return OpenAIModerationResponse.model_validate(response.json())
 
     def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None:
         """

litellm/proxy/hooks/model_budget_shared_windows.py

@@ -0,0 +1,176 @@
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
+
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.proxy.db.budget_window_spend_writer import open_window_spend_row
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+    WindowSpendTransaction,
+    build_window_spend_transaction,
+)
+from litellm.repositories.table_repositories import BudgetWindowSpendRepository
+from litellm.router_strategy.budget_limiter import budget_window_seconds
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+MODEL_BUDGET_WINDOW_ENTITY_TYPE: Final = "model_max_budget"
+_READ_TTL_SECONDS: Final = 5
+
+
+@dataclass(frozen=True, slots=True)
+class SharedWindow:
+    started_at: float
+    spend: float
+
+    def is_open(self, budget_duration: str, now: float) -> bool:
+        return now < self.started_at + budget_window_seconds(self.started_at, budget_duration)
+
+
+class LocalTTLCache(Protocol):
+    def get_cache(self, key: str) -> object: ...
+
+    def set_cache(self, key: str, value: object, *, ttl: float) -> None: ...
+
+    def delete_cache(self, key: str) -> None: ...
+
+
+class SharedWindows(Protocol):
+    async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ...
+
+    async def open(self, spend_key: str, budget_duration: str, now: float) -> float | None: ...
+
+    async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ...
+
+
+PrismaClientSource: TypeAlias = Callable[[], "PrismaClient | None"]
+WindowSpendSink: TypeAlias = Callable[[WindowSpendTransaction], Awaitable[None]]
+
+
+def _proxy_prisma_client() -> "PrismaClient | None":
+    try:
+        from litellm.proxy.proxy_server import prisma_client
+    except ImportError:
+        return None
+    return prisma_client
+
+
+async def _queue_on_proxy_spend_writer(transaction: WindowSpendTransaction) -> None:
+    from litellm.proxy.proxy_server import proxy_logging_obj
+
+    await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(transaction)
+
+
+@dataclass(frozen=True, slots=True)
+class _Reading:
+    window: SharedWindow | None
+
+
+class DatabaseSharedWindows:
+    def __init__(
+        self,
+        prisma_client_source: PrismaClientSource = _proxy_prisma_client,
+        sink: WindowSpendSink = _queue_on_proxy_spend_writer,
+        read_ttl_seconds: int = _READ_TTL_SECONDS,
+    ) -> None:
+        self._prisma_client_source = prisma_client_source
+        self._sink = sink
+        self._read_ttl_seconds = read_ttl_seconds
+        self._readings: LocalTTLCache = InMemoryCache(max_size_in_memory=10_000, default_ttl=read_ttl_seconds)
+
+    async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+        cached: Final = self._readings.get_cache(key=spend_key)
+        if isinstance(cached, _Reading):
+            return cached.window
+        window: Final = await self._read_row(spend_key=spend_key, budget_duration=budget_duration)
+        self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds)
+        return window
+
+    async def open(self, spend_key: str, budget_duration: str, now: float) -> float | None:
+        """When the window every replica shares started, opening one at `now` if none is open.
+
+        Without a database every window is this replica's own, so it starts at `now`. None means the database did
+        not answer, so this replica cannot know which window the others are in yet.
+        """
+        prisma_client: Final = self._prisma_client_source()
+        if prisma_client is None:
+            return now
+        try:
+            seen: Final = await _fetch_row(
+                prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
+            )
+            if seen is not None and seen.is_open(budget_duration=budget_duration, now=now):
+                return seen.started_at
+            await open_window_spend_row(
+                prisma_client=prisma_client,
+                entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+                entity_id=spend_key,
+                window_duration=budget_duration,
+                window_start=datetime.fromtimestamp(now, tz=timezone.utc),
+                replaces_window_start=None
+                if seen is None
+                else datetime.fromtimestamp(seen.started_at, tz=timezone.utc),
+            )
+            opened: Final = await _fetch_row(
+                prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
+            )
+        except Exception:  # noqa: BLE001  # an unreachable database leaves this replica on its own window
+            verbose_proxy_logger.exception("Unable to open the shared model budget window")
+            return None
+        if opened is None or not opened.is_open(budget_duration=budget_duration, now=now):
+            return None
+        return opened.started_at
+
+    async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+        if self._prisma_client_source() is None:
+            return
+        try:
+            await self._sink(
+                build_window_spend_transaction(
+                    entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+                    entity_id=spend_key,
+                    window_duration=budget_duration,
+                    window_start=datetime.fromtimestamp(started_at, tz=timezone.utc),
+                    spend=spend,
+                )
+            )
+        except Exception:  # noqa: BLE001  # spend tracking must never fail the success callback
+            verbose_proxy_logger.debug("Unable to queue shared model budget spend", exc_info=True)
+
+    async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+        prisma_client: Final = self._prisma_client_source()
+        if prisma_client is None:
+            return None
+        try:
+            return await _fetch_row(
+                prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=False
+            )
+        except Exception:  # noqa: BLE001  # an unreadable row leaves enforcement on this replica's own counter
+            verbose_proxy_logger.exception("Unable to read shared model budget spend")
+            return None
+
+
+async def _fetch_row(
+    prisma_client: "PrismaClient", spend_key: str, budget_duration: str, use_writer: bool
+) -> SharedWindow | None:
+    row: Final = await BudgetWindowSpendRepository(prisma_client, use_writer=use_writer).table.find_unique(
+        where={
+            "entity_type_entity_id_window_duration": {
+                "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+                "entity_id": spend_key,
+                "window_duration": budget_duration,
+            }
+        }
+    )
+    if row is None:
+        return None
+    window_start: Final = row.window_start
+    started_at: Final = (
+        window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
+    ).timestamp()
+    return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
+
+
+PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows()

litellm/proxy/litellm_pre_call_utils.py

@@ -118,6 +118,9 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
 
 _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
 
+SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority"
+ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"})
+
 # W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/
 # e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
 _TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
@@ -688,7 +691,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t
 
 
 def _strip_untrusted_request_header_controls(
-    headers: Any,
+    headers: object,
     *,
     allow_client_message_redaction_opt_out: bool = False,
 ) -> None:
@@ -1692,7 +1695,7 @@ class LiteLLMProxyRequestSetup:
 
     @staticmethod
     def add_internal_user_from_user_mapping(
-        general_settings: dict | None,
+        general_settings: Mapping[str, object] | None,
         user_api_key_dict: UserAPIKeyAuth,
         headers: dict,
     ) -> UserAPIKeyAuth:
@@ -1711,7 +1714,7 @@ class LiteLLMProxyRequestSetup:
         return user_api_key_dict
 
     @staticmethod
-    def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
+    def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None:
         """
         Get the user from the specified header if `general_settings.user_header_name` is set.
         """
@@ -1732,7 +1735,9 @@ class LiteLLMProxyRequestSetup:
         return user
 
     @staticmethod
-    def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
+    def get_openai_org_id_from_headers(
+        headers: dict, general_settings: Mapping[str, object] | None = None
+    ) -> str | None:
         """
         Get the OpenAI Org ID from the headers.
         """
@@ -1823,7 +1828,7 @@ class LiteLLMProxyRequestSetup:
         headers: dict,
         request_data: Mapping[str, object],
         user_api_key_dict: UserAPIKeyAuth,
-        general_settings: dict[str, object] | None = None,
+        general_settings: Mapping[str, object] | None = None,
     ) -> LitellmDataForBackendLLMCall:
         """
         - Adds user from headers

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -231,9 +231,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa
     return _ShadowEvalAttemptRepository(prisma_client).table
 
 
+_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
+
+
 async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]:
     async with db_span("auto_router_report_query", sql_relation(query)):
-        return await prisma_client.db.query_raw(query, *args)
+        return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args))
 
 
 async def _execute_raw(prisma_client: "PrismaClient", query: str, *args: object) -> int:

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -3,6 +3,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence
 from collections.abc import Set as AbstractSet
 from dataclasses import dataclass, replace
 from datetime import date, datetime, timedelta
+from itertools import groupby
 from types import MappingProxyType
 from typing import Final, Literal, NoReturn, Protocol
 
@@ -28,6 +29,10 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
     DailySpendData,
     DailySpendMetadata,
     GroupedData,
+    GuardrailKeyMetric,
+    GuardrailMetadata,
+    GuardrailMetric,
+    GuardrailSpendMetrics,
     KeyMetadata,
     KeyMetricWithMetadata,
     MetricWithMetadata,
@@ -39,6 +44,7 @@ from litellm.types.repositories.daily_activity import (
     DailyActivityTable,
     EntityRollupRow,
     GroupingSetsRow,
+    GuardrailSpendRow,
     KeyMetadataRow,
     RollupMetricsRow,
     SpendLogsWindow,
@@ -1013,17 +1019,68 @@ def _fold_entity_rollups_sync(
             )
 
 
+def _guardrail_metric(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> GuardrailMetric:
+    return GuardrailMetric(
+        metrics=GuardrailSpendMetrics(
+            spend=sum(row.spend for row in rows),
+            untracked_units=sum(row.untracked_units for row in rows),
+        ),
+        metadata=GuardrailMetadata(guardrail_name=rows[0].guardrail_name),
+        api_key_breakdown={
+            row.api_key: GuardrailKeyMetric(
+                metrics=GuardrailSpendMetrics(spend=row.spend, untracked_units=row.untracked_units),
+                metadata=day.breakdown.api_keys[row.api_key].metadata,
+            )
+            for row in rows
+            if row.api_key in day.breakdown.api_keys
+        },
+    )
+
+
+def _day_with_guardrails(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> DailySpendData:
+    by_guardrail: Final = groupby(sorted(rows, key=lambda row: row.guardrail_id), key=lambda row: row.guardrail_id)
+    guardrails: Final = {guardrail_id: _guardrail_metric(day, tuple(group)) for guardrail_id, group in by_guardrail}
+    return day.model_copy(update={"breakdown": day.breakdown.model_copy(update={"guardrails": guardrails})})
+
+
+def _with_guardrail_breakdown(
+    results: Sequence[DailySpendData], rows: Sequence[GuardrailSpendRow]
+) -> tuple[DailySpendData, ...]:
+    rows_by_date: Final = MappingProxyType(
+        {day: tuple(group) for day, group in groupby(sorted(rows, key=lambda row: row.date), key=lambda row: row.date)}
+    )
+    return tuple(_day_with_guardrails(day, rows_by_date.get(day.date.strftime("%Y-%m-%d"), ())) for day in results)
+
+
+async def _guardrail_spend_or_empty(
+    repository: DailyActivityRepository, scope: DailyActivityScope
+) -> tuple[GuardrailSpendRow, ...]:
+    from prisma.errors import PrismaError
+
+    try:
+        return await repository.guardrail_spend(scope)
+    except PrismaError as exc:
+        verbose_proxy_logger.warning("Returning daily activity without its guardrail breakdown: %s", exc)
+        return ()
+
+
 async def get_daily_activity_aggregated(
     repository: DailyActivityRepository,
     scope: DailyActivityScope,
     *,
     entity_metadata_field: Mapping[str, dict[str, object]] | None = None,
     include_entity_breakdown: bool = False,
+    include_guardrail_breakdown: bool = False,
     api_key_limit: int = constants.USAGE_TOP_API_KEYS_DEFAULT,
 ) -> SpendAnalyticsPaginatedResponse:
     try:
-        aggregated_rows: Final = await repository.aggregated(
-            scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
+        aggregated_rows, guardrail_rows = await asyncio.gather(
+            repository.aggregated(
+                scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
+            ),
+            _guardrail_spend_or_empty(repository, scope)
+            if include_guardrail_breakdown and not scope.model
+            else asyncio.sleep(0, result=()),
         )
         records: Final = aggregated_rows.grouping_rows
         aggregated: Final = await _aggregate_grouping_sets_records(
@@ -1058,8 +1115,13 @@ async def get_daily_activity_aggregated(
                 api_key_metadata=entity_key_metadata,
                 entity_metadata_field=entity_metadata_field,
             )
+        results: Final = (
+            list(await asyncio.to_thread(_with_guardrail_breakdown, aggregated["results"], guardrail_rows))
+            if guardrail_rows
+            else aggregated["results"]
+        )
         return SpendAnalyticsPaginatedResponse(
-            results=aggregated["results"],
+            results=results,
             metadata=DailySpendMetadata(
                 total_spend=aggregated["totals"].spend,
                 total_flat_cost=aggregated["totals"].flat_cost,

litellm/proxy/management_endpoints/daily_activity_routes.py

@@ -300,6 +300,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver,
                 resolved.scope,
                 entity_metadata_field=resolved.entity_metadata,
                 include_entity_breakdown=resolver.include_entity_breakdown,
+                include_guardrail_breakdown=resolver.include_guardrail_breakdown,
                 api_key_limit=api_key_limit,
             )
             if resolver.entity != "team":

litellm/proxy/management_endpoints/daily_activity_scopes.py

@@ -56,6 +56,7 @@ class EntityScopeResolver:
     resolve: EntityScopeResolve
     include_entity_breakdown: bool
     operation_names: OperationNames
+    include_guardrail_breakdown: bool = False
 
 
 def _query_ids(value: str | None) -> tuple[str, ...] | None:
@@ -364,6 +365,7 @@ USER_RESOLVER = EntityScopeResolver(
     query=_user_query,
     resolve=_resolve_user,
     include_entity_breakdown=False,
+    include_guardrail_breakdown=True,
     operation_names=MappingProxyType(
         {
             "aggregated": "get_user_daily_activity_aggregated",
@@ -385,6 +387,7 @@ TEAM_RESOLVER = EntityScopeResolver(
     query=_team_query,
     resolve=_resolve_team,
     include_entity_breakdown=True,
+    include_guardrail_breakdown=True,
     operation_names=MappingProxyType(
         {
             "aggregated": "get_team_daily_activity_aggregated",
@@ -423,6 +426,7 @@ ORGANIZATION_RESOLVER = EntityScopeResolver(
     query=_organization_query,
     resolve=_resolve_organization,
     include_entity_breakdown=True,
+    include_guardrail_breakdown=True,
     operation_names=MappingProxyType(
         {
             "aggregated": "get_organization_daily_activity_aggregated",

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -499,7 +499,7 @@ async def new_user(
     - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
     - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
     - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
-    - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
+    - model_max_budget: Optional[dict] - Model-specific max budget for user, keyed by model name, `*` pattern, or model access group. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
     - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user.
@@ -1702,7 +1702,7 @@ async def user_update(
         - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
         - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
         - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
-        - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
+        - model_max_budget: Optional[dict] - Model-specific max budget for user, keyed by model name, `*` pattern, or model access group. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
         - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
         - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
         - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user.

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -2328,7 +2328,7 @@ async def generate_key_fn(
     - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely.
     - enable_prompt_caching: Optional[bool] - Auto-inject prompt caching breakpoints (Anthropic cache_control markers) on requests made with this key. Supported Claude models on Anthropic, Bedrock, Vertex AI, and Azure AI only.
     - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
-    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
+    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
     - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
@@ -2538,7 +2538,7 @@ async def generate_service_account_key_fn(
     - metadata: Optional[dict] - Metadata for key, store information for key. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
     - guardrails: Optional[List[str]] - List of active guardrails for the key
     - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
-    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
+    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
     - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
@@ -3855,7 +3855,7 @@ async def update_key_fn(
     - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
     - spend: Optional[float] - Amount spent by key
     - max_budget: Optional[float] - Max budget for key
-    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
+    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
     - soft_budget: Optional[float] - Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached. Set to null to remove the soft budget.
@@ -6303,7 +6303,7 @@ async def regenerate_key_fn(
         - tags: Optional[List[str]] - Tags for organizing keys (Enterprise only)
         - spend: Optional[float] - Amount spent by key
         - max_budget: Optional[float] - Max budget for key
-        - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
+        - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
         - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
         - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
         - soft_budget: Optional[float] - Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached.

litellm/proxy/management_endpoints/router_settings_endpoints.py

@@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U
 import inspect
 from collections.abc import Mapping
 from types import MappingProxyType
-from typing import Any, Final, get_args
+from typing import Final, get_args
 
 from fastapi import APIRouter, Depends
 from pydantic import Field
@@ -32,7 +32,7 @@ router: Final = APIRouter()
 
 class RouterSettingsResponse(LiteLLMBaseModel):
     fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata")
-    current_values: dict[str, Any] = Field(description="Current values of router settings")
+    current_values: dict[str, object] = Field(description="Current values of router settings")
     routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
     routing_group_strategies: tuple[str, ...] = Field(
         description="Strategies supported when constructing a routing group"

litellm/proxy/management_endpoints/team_endpoints.py

@@ -11,6 +11,7 @@ All /team management endpoints
 
 import asyncio
 import copy
+import itertools
 import json
 import math
 import traceback
@@ -139,7 +140,12 @@ from litellm.proxy.common_utils.user_api_key_cache import (
 from litellm.proxy.db.db_span import db_span
 from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
 from litellm.proxy.hooks.model_max_budget_limiter import (
+    RoutedModels,
     build_model_max_budget_usage,
+    entries_outranked_by,
+    entry_names_model,
+    is_enforced_budget_entry,
+    proxy_routed_models,
     resolve_model_budget,
 )
 from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied
@@ -1467,8 +1473,10 @@ def _check_team_model_budget_update_authority(
     data: UpdateTeamRequest,
     user_api_key_dict: UserAPIKeyAuth,
     existing_model_max_budget: Mapping[str, object] | None,
+    routed_models_source: Callable[[str | None], RoutedModels] = proxy_routed_models,
 ) -> None:
-    """Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
+    """Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap, or
+    add an entry that takes precedence over one."""
     if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
         return
     if "model_max_budget" not in data.model_fields_set or not existing_model_max_budget:
@@ -1487,29 +1495,97 @@ def _check_team_model_budget_update_authority(
                 )
             },
         )
-    for model_name, proposed in requested.items():
-        governing = resolve_model_budget(model=model_name, model_max_budget=existing_model_max_budget)
-        if governing is None:
-            continue
-        cap = governing.budget_config
-        if cap.max_budget is None:
-            continue
-        if (
-            proposed.max_budget is None
-            or proposed.max_budget > cap.max_budget
-            or proposed.budget_duration != cap.budget_duration
-        ):
-            raise HTTPException(
-                status_code=403,
-                detail={
-                    "error": (
-                        f"Only a proxy admin can raise a team's model_max_budget for {model_name!r} or change its "
-                        f"budget_duration. Current max_budget={cap.max_budget} per {cap.budget_duration} "
-                        f"(entry {governing.budget_model!r}), requested={proposed.max_budget} per "
-                        f"{proposed.budget_duration}."
-                    )
-                },
+    routed_models: Final = routed_models_source(data.team_id)
+    for entry, proposed in requested.items():
+        existing = _existing_model_cap(existing_model_max_budget.get(entry))
+        if existing is not None:
+            _check_existing_model_cap_kept(entry=entry, cap_entry=entry, proposed=proposed, existing=existing)
+            if is_enforced_budget_entry(existing) or not is_enforced_budget_entry(proposed):
+                continue
+        governing = resolve_model_budget(model=entry, model_max_budget=existing_model_max_budget, team_id=data.team_id)
+        if governing is not None and entry_names_model(entry=governing.budget_model, model=entry):
+            _check_existing_model_cap_kept(
+                entry=entry, cap_entry=governing.budget_model, proposed=proposed, existing=governing.budget_config
             )
+        _check_new_entry_outranks_no_cap(
+            entry=entry,
+            existing_model_max_budget=existing_model_max_budget,
+            routed_models=routed_models,
+            team_id=data.team_id,
+        )
+
+
+def _check_existing_model_cap_kept(entry: str, cap_entry: str, proposed: BudgetConfig, existing: BudgetConfig) -> None:
+    if existing.max_budget is None or not _looser_model_cap(proposed=proposed, cap=existing):
+        return
+    raise HTTPException(
+        status_code=403,
+        detail={
+            "error": (
+                f"Only a proxy admin can raise a team's model_max_budget for {entry!r} or change its "
+                f"budget_duration. Current max_budget={existing.max_budget} per {existing.budget_duration} "
+                f"(entry {cap_entry!r}), requested={proposed.max_budget} per {proposed.budget_duration}."
+            )
+        },
+    )
+
+
+def _check_new_entry_outranks_no_cap(
+    entry: str,
+    existing_model_max_budget: Mapping[str, object],
+    routed_models: RoutedModels,
+    team_id: str | None,
+) -> None:
+    """A new entry gets its own counter, so the models it takes from a capped entry stop drawing from that cap."""
+    outranked: Final = next(
+        _capped_entries_outranked_by(
+            entry=entry,
+            existing_model_max_budget=existing_model_max_budget,
+            routed_models=routed_models,
+            team_id=team_id,
+        ),
+        None,
+    )
+    if outranked is None:
+        return
+    outranked_name, cap = outranked
+    raise HTTPException(
+        status_code=403,
+        detail={
+            "error": (
+                f"Only a proxy admin can add a team model_max_budget entry {entry!r} that takes precedence over "
+                f"{outranked_name!r}, since the models they share would stop drawing from its max_budget="
+                f"{cap.max_budget} per {cap.budget_duration}."
+            )
+        },
+    )
+
+
+def _capped_entries_outranked_by(
+    entry: str,
+    existing_model_max_budget: Mapping[str, object],
+    routed_models: RoutedModels,
+    team_id: str | None,
+) -> Iterator[tuple[str, BudgetConfig]]:
+    governing: Final = resolve_model_budget(model=entry, model_max_budget=existing_model_max_budget, team_id=team_id)
+    outranked_names: Final = itertools.chain(
+        ()
+        if governing is None or entry_names_model(entry=governing.budget_model, model=entry)
+        else (governing.budget_model,),
+        entries_outranked_by(entry=entry, model_max_budget=existing_model_max_budget, routed_models=routed_models),
+    )
+    outranked_caps: Final = ((name, _existing_model_cap(existing_model_max_budget[name])) for name in outranked_names)
+    yield from (
+        (name, cap) for name, cap in outranked_caps if name != entry and cap is not None and cap.max_budget is not None
+    )
+
+
+def _looser_model_cap(proposed: BudgetConfig, cap: BudgetConfig) -> bool:
+    return (
+        proposed.max_budget is None
+        or (cap.max_budget is not None and proposed.max_budget > cap.max_budget)
+        or (is_enforced_budget_entry(cap) and proposed.budget_duration != cap.budget_duration)
+    )
 
 
 def _should_auto_add_team_creator(
@@ -1572,7 +1648,7 @@ async def new_team(
     - prompts: Optional[List[str]] - List of prompts that the team is allowed to use.
     - organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`.
     - model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias)
-    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
+    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model, keyed by model name, `*` pattern, or model access group. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
     - guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails)
     - policies: Optional[List[str]] - Policies for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies)
     - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the team. Proxy admin only.
@@ -2350,7 +2426,7 @@ async def update_team(
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
     - organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`.
     - model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias)
-    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
+    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model, keyed by model name, `*` pattern, or model access group. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
     - guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails)
     - policies: Optional[List[str]] - Policies for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies)
     - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the team. Proxy admin only.

litellm/proxy/management_helpers/team_roster_sync.py

@@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import (
 )
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
+from litellm.repositories.team_repository import TeamRepository
 
 if TYPE_CHECKING:
     from prisma import Prisma
@@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool:
 
 
 async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone:
-    team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id})
+    team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id})
     if team_row is None:
         return TeamGone(team_id=team_id)
     team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())

litellm/proxy/mcp_server/discoverable_endpoints.py

@@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict:
     if decrypted_json is None:
         raise ValueError("Failed to decrypt state parameter")
 
-    state_data: Final = json.loads(decrypted_json)
+    state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json))
     return state_data
 
 
 _MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_"
 _DECODED_JSON: Final = TypeAdapter(object)
+_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
 
 
 class _ManagedAuthorizationCode(LiteLLMBaseModel):

litellm/proxy/openai_files_endpoints/files_endpoints.py

@@ -6,9 +6,12 @@
 ######################################################################
 
 import asyncio
+import functools
+import itertools
 import traceback
-from collections.abc import Mapping, Sequence
-from typing import Any, BinaryIO, Final, TypedDict, cast, get_args
+from collections.abc import Iterable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, BinaryIO, Final, NoReturn, TypedDict, cast, get_args
 
 import httpx
 from fastapi import (
@@ -28,12 +31,13 @@ from typing_extensions import ReadOnly
 import litellm
 from litellm import CreateFileRequest, get_secret_str
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import ANTHROPIC_DEFAULT_FILE_LIST_LIMIT
 from litellm.litellm_core_utils.cloud_storage_security import (
     is_managed_cloud_storage_uri,
 )
 from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
 from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME
-from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
 from litellm.llms.base_llm.managed_resources.isolation import build_list_page
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@@ -112,6 +116,9 @@ from litellm.types.llms.openai import (
     OpenAIFileObject,
     OpenAIFilesPurpose,
 )
+from litellm.types.proxy.auth.user_api_key_auth import user_api_key_has_admin_view
+from litellm.types.router import Deployment
+from litellm.types.utils import LlmProviders
 
 router: Final = APIRouter()
 
@@ -1673,6 +1680,185 @@ def _as_file_list_page(response: object) -> object:
     return FileListPage.model_validate(page)
 
 
+async def _list_owned_managed_files(
+    managed_files_obj: BaseFileEndpoints,
+    user_api_key_dict: UserAPIKeyAuth,
+    purpose: str | None,
+    limit: int | None,
+    after: str | None,
+    before: str | None,
+    model_scope: FileListModelScope | None,
+) -> FileListPage:
+    return await managed_files_obj.afile_list(
+        purpose=purpose,
+        litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+        user_api_key_dict=user_api_key_dict,
+        limit=limit,
+        after=after,
+        before=before,
+        model_scope=model_scope,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _UnscopedFileList:
+    status_code: int
+    detail: str
+
+
+@dataclass(frozen=True, slots=True)
+class _ListedFiles:
+    model_scope: FileListModelScope | None
+    custom_llm_provider: str | None
+
+
+def _raise_unscoped_file_list(failure: _UnscopedFileList) -> NoReturn:
+    raise HTTPException(status_code=failure.status_code, detail=failure.detail)
+
+
+def _single_target_model_name(target_model_names: str) -> str | _UnscopedFileList:
+    target_model_names_list: Final = target_model_names.split(",")
+    if len(target_model_names_list) != 1:
+        return _UnscopedFileList(
+            status_code=400,
+            detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
+        )
+    return target_model_names_list[0]
+
+
+async def _requested_custom_llm_provider(request: Request, provider: str | None) -> str | None:
+    return (
+        provider
+        or get_custom_llm_provider_from_request_headers(request=request)
+        or get_custom_llm_provider_from_request_query(request=request)
+        or await get_custom_llm_provider_from_request_body(request=request)
+    )
+
+
+def _default_file_list_limit(custom_llm_provider: str | None) -> int | None:
+    return ANTHROPIC_DEFAULT_FILE_LIST_LIMIT if custom_llm_provider == LlmProviders.ANTHROPIC.value else None
+
+
+def _deployment_model_mapping_keys(deployment: Deployment) -> frozenset[str]:
+    return frozenset(key for key in (deployment.model_name, deployment.model_info.id) if key is not None)
+
+
+def _model_mapping_keys(deployments: Iterable[Deployment]) -> frozenset[str]:
+    return frozenset(itertools.chain.from_iterable(map(_deployment_model_mapping_keys, deployments)))
+
+
+def _deployments_serving_model(llm_router: Router, model: str) -> tuple[Deployment, ...]:
+    deployment: Final = llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) else None
+    if deployment is not None:
+        return (deployment,)
+    return tuple(Deployment.model_validate(listed) for listed in llm_router.get_model_list(model_name=model) or ())
+
+
+def _resolved_deployment_provider(deployment: Deployment) -> str | None:
+    try:
+        return litellm.get_llm_provider(
+            model=deployment.litellm_params.model,
+            custom_llm_provider=deployment.litellm_params.custom_llm_provider,
+        )[1]
+    except litellm.BadRequestError:
+        return None
+
+
+def _deployments_on_provider(llm_router: Router | None, custom_llm_provider: str) -> Iterator[Deployment]:
+    if llm_router is None:
+        return iter(())
+    return (
+        deployment
+        for deployment_id in llm_router.get_model_ids()
+        if (deployment := llm_router.get_deployment(model_id=deployment_id)) is not None
+        and _resolved_deployment_provider(deployment) == custom_llm_provider
+    )
+
+
+def _routed_model_name_on_provider(
+    llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
+) -> bool:
+    if model_name in llm_router.model_names:
+        return model_name in provider_model_names
+    return llm_router.pattern_router.routes_to_provider(model_name, custom_llm_provider)
+
+
+def _stored_model_name_routes_to_provider(
+    llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
+) -> bool:
+    routing_group: Final = llm_router.get_routing_group(model_name)
+    routed_model_names: Final = (
+        tuple(routing_group.models) if routing_group is not None else (llm_router.routable_model_group(model_name),)
+    )
+    return any(
+        _routed_model_name_on_provider(llm_router, custom_llm_provider, provider_model_names, routed_model_name)
+        for routed_model_name in routed_model_names
+    )
+
+
+def _files_on_provider(llm_router: Router | None, custom_llm_provider: str) -> _ListedFiles | _UnscopedFileList:
+    from litellm.proxy.pass_through_endpoints.managed_id_rewriter import passthrough_file_model_id
+
+    if custom_llm_provider not in litellm.provider_list:
+        return _UnscopedFileList(status_code=400, detail=f"Unknown provider '{custom_llm_provider}' for listing files.")
+    passthrough_model_id: Final = passthrough_file_model_id(custom_llm_provider)
+    deployments: Final = tuple(_deployments_on_provider(llm_router, custom_llm_provider))
+    model_mapping_keys: Final = _model_mapping_keys(deployments) | frozenset(
+        (passthrough_model_id,) if passthrough_model_id is not None else ()
+    )
+    if llm_router is None:
+        return _ListedFiles(FileListModelScope(model_mapping_keys), custom_llm_provider)
+    return _ListedFiles(
+        FileListModelScope(
+            model_mapping_keys,
+            routes_stored_model_name=functools.partial(
+                _stored_model_name_routes_to_provider,
+                llm_router,
+                custom_llm_provider,
+                frozenset(deployment.model_name for deployment in deployments),
+            ),
+        ),
+        custom_llm_provider,
+    )
+
+
+def _files_serving_model(llm_router: Router, model: str) -> _ListedFiles:
+    deployments: Final = _deployments_serving_model(llm_router, model)
+    providers: Final = frozenset(map(_resolved_deployment_provider, deployments))
+    return _ListedFiles(
+        FileListModelScope(_model_mapping_keys(deployments)), next(iter(providers)) if len(providers) == 1 else None
+    )
+
+
+async def _listed_files(
+    llm_router: Router | None,
+    user_api_key_dict: UserAPIKeyAuth,
+    custom_llm_provider: str | None,
+    target_model_names: str | None,
+    routed_model: str | None,
+) -> _ListedFiles | _UnscopedFileList:
+    if routed_model is not None and llm_router is not None:
+        return _files_serving_model(llm_router, routed_model)
+    if target_model_names:
+        target_model_name: Final = _single_target_model_name(target_model_names)
+        if isinstance(target_model_name, _UnscopedFileList):
+            return target_model_name
+        if llm_router is None:
+            return _UnscopedFileList(
+                status_code=500, detail="LLM Router not initialized. Ensure models added to proxy."
+            )
+        await get_authorized_credentials_for_model(
+            llm_router=llm_router,
+            model_id=target_model_name,
+            user_api_key_dict=user_api_key_dict,
+            operation_context="file list",
+        )
+        return _files_serving_model(llm_router, target_model_name)
+    if custom_llm_provider is None:
+        return _ListedFiles(None, None)
+    return _files_on_provider(llm_router, custom_llm_provider)
+
+
 @router.get(
     "/{provider}/v1/files",
     dependencies=[Depends(user_api_key_auth)],
@@ -1697,6 +1883,8 @@ async def list_files(
     purpose: str | None = None,
     limit: int | None = None,
     after: str | None = None,
+    after_id: str | None = None,
+    before_id: str | None = None,
 ):
     """
     Returns information about a specific file. that can be used across - Assistants API, Batch API 
@@ -1739,6 +1927,10 @@ async def list_files(
         )
 
         response: Any | None = None
+        owned_files_hook: Final = proxy_logging_obj.get_proxy_hook("managed_files")
+        list_owned_files_only: Final = litellm.require_managed_files is True and not user_api_key_has_admin_view(
+            user_api_key_dict
+        )
 
         # Check for model-based credential routing (no file_id encoding check for list)
         should_route, model_used, _, credentials = await handle_model_based_routing(
@@ -1750,7 +1942,36 @@ async def list_files(
             check_file_id_encoding=False,
         )
 
-        if should_route and credentials is not None:
+        if list_owned_files_only:
+            if after and after_id and after != after_id:
+                raise HTTPException(
+                    status_code=400,
+                    detail="Pass the list cursor as either 'after' or 'after_id', not both with different ids.",
+                )
+            if not isinstance(owned_files_hook, BaseFileEndpoints):
+                raise HTTPException(
+                    status_code=500,
+                    detail="Managed file listing is unavailable, so require_managed_files cannot scope the list to the caller.",
+                )
+            listed_files: Final = await _listed_files(
+                llm_router=llm_router,
+                user_api_key_dict=user_api_key_dict,
+                custom_llm_provider=await _requested_custom_llm_provider(request=request, provider=provider),
+                target_model_names=target_model_names,
+                routed_model=model_used if should_route else None,
+            )
+            if isinstance(listed_files, _UnscopedFileList):
+                _raise_unscoped_file_list(listed_files)
+            response = await _list_owned_managed_files(  # rebind-ok: each dispatch branch binds response
+                managed_files_obj=owned_files_hook,
+                user_api_key_dict=user_api_key_dict,
+                purpose=purpose,
+                limit=limit if limit is not None else _default_file_list_limit(listed_files.custom_llm_provider),
+                after=after or after_id,
+                before=before_id,
+                model_scope=listed_files.model_scope,
+            )
+        elif should_route and credentials is not None:
             # Use model-based routing with credentials from config
             prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True)
             response = await litellm.afile_list(
@@ -1762,12 +1983,9 @@ async def list_files(
             verbose_proxy_logger.debug("Listed files using model: %s", model_used)
 
         elif target_model_names and isinstance(target_model_names, str):
-            target_model_names_list: Final = target_model_names.split(",")
-            if len(target_model_names_list) != 1:
-                raise HTTPException(
-                    status_code=400,
-                    detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
-                )
+            target_model_name: Final = _single_target_model_name(target_model_names)
+            if isinstance(target_model_name, _UnscopedFileList):
+                _raise_unscoped_file_list(target_model_name)
             if llm_router is None:
                 raise HTTPException(
                     status_code=500,
@@ -1775,7 +1993,7 @@ async def list_files(
                 )
             credentials = await get_authorized_credentials_for_model(
                 llm_router=llm_router,
-                model_id=target_model_names_list[0],
+                model_id=target_model_name,
                 user_api_key_dict=user_api_key_dict,
                 operation_context="file list",
             )
@@ -1786,20 +2004,17 @@ async def list_files(
                 **data,
             )
         else:
-            custom_llm_provider: Final = (
-                provider
-                or get_custom_llm_provider_from_request_headers(request=request)
-                or get_custom_llm_provider_from_request_query(request=request)
-                or await get_custom_llm_provider_from_request_body(request=request)
-            )
+            custom_llm_provider: Final = await _requested_custom_llm_provider(request=request, provider=provider)
             managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files")
             if custom_llm_provider is None and isinstance(managed_files_obj, BaseFileEndpoints):
-                response = await managed_files_obj.afile_list(
-                    purpose=purpose,
-                    litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+                response = await _list_owned_managed_files(  # rebind-ok: each dispatch branch binds response
+                    managed_files_obj=managed_files_obj,
                     user_api_key_dict=user_api_key_dict,
+                    purpose=purpose,
                     limit=limit,
                     after=after,
+                    before=None,
+                    model_scope=None,
                 )
             else:
                 resolved_custom_llm_provider: Final = custom_llm_provider or "openai"

litellm/proxy/pass_through_endpoints/managed_id_rewriter.py

@@ -242,6 +242,12 @@ def _passthrough_sentinel_model_id(provider: str) -> str:
     return f"_passthrough_{provider}"
 
 
+def passthrough_file_model_id(provider: str) -> str | None:
+    if (provider, "/v1/files") not in _LIST_ROUTE_TABLE:
+        return None
+    return _passthrough_sentinel_model_id(provider)
+
+
 # Key under which the provider marker is stored in a file row's model_mappings.
 # Its value lands in flat_model_file_ids (built from model_mappings.values()),
 # giving the file table a DB-queryable provider scope it otherwise lacks.

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -30,6 +30,7 @@ from pydantic import TypeAdapter
 from starlette.datastructures import UploadFile as StarletteUploadFile
 from starlette.routing import BaseRoute, Route
 from starlette.websockets import WebSocketState
+from typing_extensions import ReadOnly
 from websockets.asyncio.client import connect
 from websockets.exceptions import (
     ConnectionClosedError,
@@ -173,8 +174,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool:
     return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
 
 
+class _RegisteredPassThroughRoute(TypedDict):
+    endpoint_id: ReadOnly[str]
+    path: ReadOnly[str | None]
+    type: ReadOnly[str]
+    methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]]
+    auth: ReadOnly[bool]
+    passthrough_params: ReadOnly[Mapping[str, object]]
+
+
 # Global registry to track registered pass-through routes and prevent memory leaks
-_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
+_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {}
 
 
 def get_response_body(response: httpx.Response) -> dict | None:
@@ -2239,7 +2249,7 @@ def create_pass_through_route(
                     status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
                     detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.",
                 )
-            target_params: Final = {
+            target_params: Final[dict[str, object]] = {
                 "target": target,
                 "custom_headers": custom_headers,
                 "forward_headers": _forward_headers,
@@ -2271,7 +2281,9 @@ def create_pass_through_route(
 
             # Ensure custom_headers is a dict. Botocore returns a HeadersDict
             # for SigV4-prepared requests, which is a Mapping but not a dict.
-            headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {}
+            headers_dict: Final[dict[str, object]] = (
+                {**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {}
+            )
 
             # Ensure query_params and custom_body are dicts or None
             final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {}
@@ -3104,9 +3116,7 @@ class SafeRouteAdder:
                 return f"{path}/{{subpath:path}}"
             return None
 
-        def registered_route_methods(
-            route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]],
-        ) -> tuple[tuple[str, str], ...]:
+        def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]:
             starlette_path: Final = registered_route_path(route_info)
             methods: Final = route_info.get("methods")
             if starlette_path is None or not isinstance(methods, (list, tuple)):
@@ -3462,7 +3472,7 @@ class InitPassThroughEndpointHelpers:
         return (comparison_route, *forwarded)
 
     @staticmethod
-    def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None:
+    def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None:
         """Get passthrough params for a given route and optionally filter by HTTP method"""
         comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
         for key in _registered_pass_through_routes:

litellm/proxy/proxy_server.py

@@ -666,7 +666,11 @@ from litellm.proxy.list_api.common import (
     problem_response,
     request_validation_problem,
 )
-from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
+from litellm.proxy.litellm_pre_call_utils import (
+    SCHEDULER_PRIORITY_HEADER,
+    LiteLLMProxyRequestSetup,
+    add_litellm_data_to_request,
+)
 from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
     rust_control_plane_router,
 )
@@ -18491,6 +18495,11 @@ async def async_queue_request(
             raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value})
         router: Final = llm_router
 
+        data.update(
+            router.priority_header_update(
+                model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
+            )
+        )
         queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data)
         if "stream" in data and data["stream"] is True:  # use generate_responses to stream responses
 

litellm/proxy/schema.prisma

@@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
   @@index([unified_file_id])
   @@index([flat_model_file_ids], type: Gin)
   @@index([team_id, created_at(sort: Desc)])
+  @@index([created_by, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
   @@index([window_end])
   @@index([created_at])
 }
+
+enum AccountingEventKind {
+  inbound_usage
+  outbound_usage
+  cost
+}
+
+enum AccountingDeliveryState {
+  pending
+  applied
+  parked
+}
+
+enum AccountingDeliveryTarget {
+  LiteLLM_VerificationToken
+  LiteLLM_UserTable
+  LiteLLM_TeamTable
+  LiteLLM_TeamMembership
+  LiteLLM_OrganizationTable
+  LiteLLM_OrganizationMembership
+  LiteLLM_EndUserTable
+  LiteLLM_AgentsTable
+  LiteLLM_TagTable
+  LiteLLM_ProjectTable
+  LiteLLM_ModelAccessGroupBudgetTable
+  LiteLLM_BudgetTable
+  LiteLLM_BudgetWindowSpend
+  LiteLLM_DailyUserSpend
+  LiteLLM_DailyTeamSpend
+  LiteLLM_DailyOrganizationSpend
+  LiteLLM_DailyEndUserSpend
+  LiteLLM_DailyAgentSpend
+  LiteLLM_DailyTagSpend
+  LiteLLM_DailyModelUsage
+  LiteLLM_DailyToolSpend
+  LiteLLM_AutoRouterDailySpend
+}
+
+// Durable accounting ledger. One row per usage or cost event; payload holds the
+// private event envelope versioned by schema_version. Usage rows retain request-scoped
+// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
+model LiteLLM_AccountingEvent {
+  event_id        String              @id
+  kind            AccountingEventKind
+  source_event_id String?
+  schema_version  Int
+  payload_hash    String
+  occurred_at     DateTime
+  recorded_at     DateTime            @default(now())
+  payload         Json
+
+  @@index([recorded_at])
+  @@index([kind, source_event_id])
+}
+
+// One row per (event, target table, version): pending work for the projector and,
+// once applied in the same transaction as the target write, the receipt that stops a retry.
+model LiteLLM_AccountingDelivery {
+  event_id        String
+  target          AccountingDeliveryTarget
+  version         Int
+  state           AccountingDeliveryState  @default(pending)
+  attempts        Int                      @default(0)
+  next_attempt_at DateTime                 @default(now())
+  applied_at      DateTime?
+  last_error      String?
+
+  @@id([event_id, target, version])
+  @@index([target, state, next_attempt_at])
+}

litellm/proxy/spend_tracking/key_metadata_recovery.py

@@ -6,7 +6,7 @@ from datetime import datetime, timedelta
 from types import MappingProxyType
 from typing import Final, Literal, TypeVar
 
-from pydantic import TypeAdapter
+from pydantic import ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
@@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel):
     last_owner: str | None = None
 
 
+_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
 _TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
 _SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...])
 _DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...])
@@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout(
         await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL)
         for setting in planner_settings:
             await transaction.execute_raw(setting)
-        return await transaction.query_raw(sql, *params)
+        return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params))
 
 
 async def _reverse_hash_key_metadata(

litellm/repositories/daily_activity_repository.py

@@ -19,6 +19,7 @@ from litellm.repositories.daily_activity_sql import (
     build_cache_leakage_keys_sql,
     build_entity_rollup_sql,
     build_export_sql,
+    build_guardrail_spend_sql,
     build_key_page_sql,
     build_key_search_sql,
     build_model_top_keys_sql,
@@ -36,6 +37,7 @@ from litellm.types.repositories.daily_activity import (
     ExportRow,
     ExportType,
     GroupingSetsRow,
+    GuardrailSpendRow,
     KeyMetadataRow,
     KeyPage,
     KeySpendRow,
@@ -100,6 +102,7 @@ class DailyActivityDatabase(Protocol):
 
 _GROUPING_ADAPTER: Final = TypeAdapter(tuple[GroupingSetsRow, ...])
 _ENTITY_ADAPTER: Final = TypeAdapter(tuple[EntityRollupRow, ...])
+_GUARDRAIL_SPEND_ADAPTER: Final = TypeAdapter(tuple[GuardrailSpendRow, ...])
 _KEY_SPEND_ADAPTER: Final = TypeAdapter(tuple[KeySpendRow, ...])
 _KEY_PAGE_TOTAL_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(int)
 _USER_SPEND_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...])
@@ -191,6 +194,9 @@ class DailyActivityRepository:
             distinct_api_keys=distinct_api_keys,
         )
 
+    async def guardrail_spend(self, scope: DailyActivityScope) -> tuple[GuardrailSpendRow, ...]:
+        return _GUARDRAIL_SPEND_ADAPTER.validate_python(await self._query(build_guardrail_spend_sql(scope)))
+
     async def search_keys(self, scope: DailyActivityScope, *, search: str, limit: int) -> tuple[str, ...]:
         if not 1 <= limit <= constants.USAGE_KEY_SEARCH_MAX:
             raise ValueError(f"limit must be between 1 and {constants.USAGE_KEY_SEARCH_MAX}")

litellm/repositories/daily_activity_sql.py

@@ -261,6 +261,25 @@ def build_entity_rollup_sql(scope: DailyActivityScope, *, api_key_limit: int) ->
     return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, api_key_limit))
 
 
+def build_guardrail_spend_sql(scope: DailyActivityScope) -> SqlQuery:
+    pg_table: Final = PRISMA_TO_PG_TABLE[scope.table]
+    where_clause, where_params = build_where_clause(scope)
+    sql: Final = f"""
+        SELECT g.date, g.guardrail_id, COALESCE(t.guardrail_name, g.guardrail_id) AS guardrail_name, g.api_key,
+               COALESCE(SUM(g.cost), 0)::float AS spend,
+               COALESCE(SUM(CASE WHEN g.cost IS NULL THEN g.units ELSE g.untracked_units END), 0)::bigint AS untracked_units
+        FROM "LiteLLM_DailyGuardrailUsageUnits" g
+        LEFT JOIN "LiteLLM_GuardrailsTable" t ON t.guardrail_id = g.guardrail_id
+        WHERE g.date >= $1 AND g.date <= $2
+          AND EXISTS (
+              SELECT 1 FROM "{pg_table}"
+              WHERE {where_clause} AND "{pg_table}".date = g.date AND "{pg_table}".api_key = g.api_key
+          )
+        GROUP BY g.date, g.guardrail_id, t.guardrail_name, g.api_key
+    """
+    return SqlQuery(sql=sql, params=where_params)
+
+
 def _key_spend_select() -> str:
     return """
             COALESCE(SUM(spend), 0)::float AS spend,

litellm/responses/streaming_iterator.py

@@ -41,7 +41,11 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
-from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation
+from litellm.litellm_core_utils.internal_call_metadata import (
+    is_background_response,
+    requests_background,
+    without_budget_reservation,
+)
 from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
@@ -65,7 +69,7 @@ from litellm.types.llms.openai import (
     ResponsesAPIStreamingResponse,
 )
 from litellm.types.responses.main import GenericResponseOutputItem, OutputText
-from litellm.types.utils import CallTypes
+from litellm.types.utils import CallTypes, ModelResponse
 from litellm.utils import async_post_call_success_deployment_hook
 
 if TYPE_CHECKING:
@@ -288,7 +292,7 @@ def stream_error_status_and_message(error_obj: object) -> tuple[int, str]:
     return _status_code_for_error_fields(error_type, error_code), message
 
 
-def _map_stream_error_to_exception(
+def map_stream_error_to_exception(
     error_obj: object,
     model: str,
     custom_llm_provider: str,
@@ -578,6 +582,7 @@ class BaseResponsesAPIStreamingIterator:
                                     self.request_data.get("input"),
                                     self.request_data,
                                     self._generated_content + self._generated_tool_arguments,
+                                    use_default_image_token_count=False,
                                 )
                                 if _estimate_wanted
                                 else None
@@ -731,7 +736,7 @@ class BaseResponsesAPIStreamingIterator:
         self._earlier_rounds_spend = earlier
 
     def _map_error_event_exception(self, error_obj: object) -> Exception:
-        return _map_stream_error_to_exception(
+        return map_stream_error_to_exception(
             error_obj,
             self.model or "",
             self.custom_llm_provider or "",
@@ -981,6 +986,39 @@ class BaseResponsesAPIStreamingIterator:
         except Exception:
             pass
 
+    def _record_partial_usage_for_failure(self) -> None:
+        generated_text: Final = self._generated_content + self._generated_tool_arguments
+        if (
+            not generated_text
+            or requests_background(self.request_data)
+            or is_background_response(self._created_response)
+            or self._completed_response_logged
+            or self.logging_obj.model_call_details.get("combined_usage_object") is not None
+        ):
+            return
+        estimate: Final = _estimate_usage_safely(
+            self.model or "",
+            self.request_data.get("input"),
+            self.request_data,
+            generated_text,
+            use_default_image_token_count=True,
+        )
+        if estimate is None:
+            return
+        usage: Final = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(  # pyright: ignore[reportPrivateUsage]  # same shared transform the success path above uses
+            estimate
+        )
+        served_tier: Final[object] = getattr(self._created_response, "service_tier", None)
+        assembled: Final = ModelResponse(
+            model=self.model, usage=usage, service_tier=served_tier if isinstance(served_tier, str) else None
+        )
+        if self._earlier_rounds_spend is None:
+            self.logging_obj.record_assembled_response_for_failure(assembled)
+            return
+        self.logging_obj.record_failure_after_earlier_rounds(
+            self._earlier_rounds_spend, usage, self.logging_obj.response_cost_calculator(result=assembled) or 0.0
+        )
+
     def _handle_failure(self, exception: Exception):
         """
         Trigger failure handlers before bubbling the exception.
@@ -990,6 +1028,7 @@ class BaseResponsesAPIStreamingIterator:
         if self._failure_handled:
             return
         self._failure_handled = True
+        self._record_partial_usage_for_failure()
 
         traceback_exception: Final = traceback.format_exc()
         end_time: Final = datetime.now()
@@ -1060,15 +1099,20 @@ class BaseResponsesAPIStreamingIterator:
         the same shape the success handler assembles for a natural end of stream.
         Its response is the response.created payload (it carries the served
         service_tier) plus the text and tool arguments relayed so far, with counted
-        usage since the provider never sent any. None when nothing was relayed yet
-        or the stream already reached its terminal event and billed itself.
+        usage since the provider never sent any. None when nothing was relayed yet,
+        the stream already reached its terminal event and billed itself, or its failure
+        log already billed the relayed output.
         """
         created: Final = self._created_response
         generated: Final = self._generated_content + self._generated_tool_arguments
-        if self.completed_response is not None or created is None or not generated:
+        if created is None or not generated or self._billed_without_disconnect():
             return None
         usage: Final = _estimate_usage_safely(
-            self.model or "", self.request_data.get("input"), self.request_data, generated
+            self.model or "",
+            self.request_data.get("input"),
+            self.request_data,
+            generated,
+            use_default_image_token_count=False,
         )
         if usage is None:
             return None
@@ -1082,6 +1126,9 @@ class BaseResponsesAPIStreamingIterator:
         stamp_responses_usage_cost(partial, self.logging_obj)
         return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
 
+    def _billed_without_disconnect(self) -> bool:
+        return self.completed_response is not None or self._failure_handled
+
     def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None:
         self._yielded_first_chunk = True
         if event.type not in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES:
@@ -1218,6 +1265,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
                 raise StopAsyncIteration from e
             self._raise_for_transport_error(e)
         except httpx.TimeoutException as e:
+            if self.completed_response is not None:
+                self.finished = True
+                raise StopAsyncIteration from e
             self._raise_for_transport_timeout(e)
         except httpx.HTTPError as e:
             # Handle HTTP errors
@@ -1303,6 +1353,9 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
                 raise StopIteration from e
             self._raise_for_transport_error(e)
         except httpx.TimeoutException as e:
+            if self.completed_response is not None:
+                self.finished = True
+                raise StopIteration from e
             self._raise_for_transport_timeout(e)
         except httpx.HTTPError as e:
             # Handle HTTP errors
@@ -1727,6 +1780,8 @@ def _estimate_usage_from_text(
     request_input: object,
     responses_api_request: Mapping[str, object],
     generated_text: str,
+    *,
+    use_default_image_token_count: bool = False,
 ) -> ResponseAPIUsage:
     messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(  # pyright: ignore[reportUnknownMemberType]  # the transformer's signature is partially untyped
         input=request_input,  # pyright: ignore[reportArgumentType]  # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union
@@ -1734,7 +1789,7 @@ def _estimate_usage_from_text(
         only_since_compaction=True,
     )
     input_tokens: Final = litellm.token_counter(  # pyright: ignore[reportUnknownMemberType]  # token_counter's public signature is untyped
-        model=model, messages=messages
+        model=model, messages=messages, use_default_image_token_count=use_default_image_token_count
     )
     output_tokens: Final = litellm.token_counter(  # pyright: ignore[reportUnknownMemberType]  # token_counter's public signature is untyped
         model=model, text=generated_text, count_response_tokens=True
@@ -1751,6 +1806,8 @@ def _estimate_usage_safely(
     request_input: object,
     responses_api_request: Mapping[str, object],
     generated_text: str,
+    *,
+    use_default_image_token_count: bool,
 ) -> ResponseAPIUsage | None:
     try:
         return _estimate_usage_from_text(
@@ -1758,6 +1815,7 @@ def _estimate_usage_safely(
             request_input=request_input,
             responses_api_request=responses_api_request,
             generated_text=generated_text,
+            use_default_image_token_count=use_default_image_token_count,
         )
     except Exception as e:
         verbose_logger.debug("Could not estimate usage from stream text, billing $0: %s", e)
@@ -2276,7 +2334,7 @@ class _ResponsesWsTurn:
         )
         if failed_event is None:
             return None
-        return _map_stream_error_to_exception(
+        return map_stream_error_to_exception(
             _ws_event_error(failed_event), model, custom_llm_provider, logging_obj=connection_logging_obj
         )
 

litellm/router.py

@@ -359,6 +359,7 @@ from litellm.router_utils.request_priority import (
     InvalidPriority,
     parse_default_priority,
     priority_error,
+    priority_header_fields,
     request_drops_params,
     resolve_request_priority,
     unqueueable_priority_message,
@@ -1369,7 +1370,7 @@ class Router:
             },
         ]
 
-        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
+        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}])
         ```
         """
 
@@ -1595,6 +1596,10 @@ class Router:
 
         if default_fallbacks is not None or litellm.default_fallbacks is not None:
             _fallbacks = default_fallbacks or litellm.default_fallbacks
+            if isinstance(_fallbacks, str):
+                raise TypeError(
+                    f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']"
+                )
             if self.fallbacks is not None:
                 self.fallbacks.append({"*": _fallbacks})
             else:
@@ -2838,7 +2843,7 @@ class Router:
         self._initialize_core_endpoints()
         self._initialize_specialized_endpoints()
 
-    def validate_fallbacks(self, fallback_param: list | None):
+    def validate_fallbacks(self, fallback_param: Sequence[Mapping[str, object]] | None):
         """
         Validate the fallbacks parameter.
         """
@@ -2851,6 +2856,13 @@ class Router:
                 raise ValueError(
                     f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys."
                 )
+            ((model_group, targets),) = fallback_dict.items()
+            if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=[fallback_dict]):
+                continue
+            raise TypeError(
+                f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. "
+                f"Use {{'{model_group}': ['{targets}']}}"
+            )
 
     def _add_encrypted_content_affinity_check(self, enable_global_affinity: bool) -> None:
         def _move_before_deployment_affinity(
@@ -5411,14 +5423,22 @@ class Router:
         request_priority: Final = resolve_request_priority(
             requested=kwargs.get("priority"),
             default_priority=self.default_priority,
-            drops_params=lambda: request_drops_params(
-                kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
-            ),
+            drops_params=lambda: self._drops_params(model=model, kwargs=kwargs),
         )
         if isinstance(request_priority, InvalidPriority):
             raise priority_error(model=model, message=request_priority.message)
         return request_priority, {key: value for key, value in kwargs.items() if key != "priority"}
 
+    def priority_header_update(
+        self, model: str, kwargs: Mapping[str, object], header: str | None
+    ) -> Mapping[str, int | str]:
+        return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs))
+
+    def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool:
+        return request_drops_params(
+            kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
+        )
+
     async def _schedule_factory(
         self,
         model: str,
@@ -6456,6 +6476,9 @@ class Router:
         hop_kwargs: Final = mid_stream_fallback_hop_kwargs(
             model=model, original_generic_function=original_generic_function, controls=controls, kwargs=kwargs
         )
+        shared_logging: Final = kwargs.get("litellm_logging_obj")
+        if isinstance(shared_logging, LiteLLMLogging):
+            shared_logging.carry_failed_attempt_usage()
         response: Final = await self._ageneric_api_call_with_fallbacks_helper(
             model=model, original_generic_function=original_generic_function, **kwargs
         )
@@ -11847,6 +11870,11 @@ class Router:
             indices: Final = self.model_name_to_deployment_indices.get(model_name)
             return None if indices is None else tuple(self.model_list[idx] for idx in indices)
 
+    def team_public_model_names_of(self, team_id: str) -> frozenset[str]:
+        """The public names `team_id` serves its own deployments under."""
+        with self._deployment_index_lock:
+            return frozenset(name for owner, name in self.team_model_to_deployment_indices if owner == team_id)
+
     def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[DeploymentTypedDict, ...] | None:
         """The deployments `team_id` serves under `public_model_name`, or None when that pair is not indexed."""
         with self._deployment_index_lock:

litellm/router_utils/cooldown_handlers.py

@@ -13,7 +13,7 @@ from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm._internal_context import service_target
@@ -417,17 +417,35 @@ def _should_run_cooldown_logic(
     return True
 
 
-_FALLBACK_ENTRIES_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str | Mapping[str, object]]] | str, ...])
+_FALLBACK_ENTRIES_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+_FALLBACK_ENTRY_ADAPTER: Final = TypeAdapter(Mapping[str, list[str | Mapping[str, object]] | str])
 
 
 def _fallback_target_name(target: str | Mapping[str, object]) -> str:
     return target if isinstance(target, str) else str(target.get("model", ""))
 
 
+def _fallback_target_names(
+    targets: Sequence[str | Mapping[str, object]] | str,
+) -> list[str]:  # mutable-ok: mirrors get_fallback_model_group's fallbacks shape
+    return [targets] if isinstance(targets, str) else [_fallback_target_name(target) for target in targets]
+
+
 def _fallback_chain_names(
-    entry: Mapping[str, Sequence[str | Mapping[str, object]]],
-) -> Mapping[str, list[str]]:  # mutable-ok: mirrors get_fallback_model_group's fallbacks shape
-    return {key: [_fallback_target_name(target) for target in targets] for key, targets in entry.items()}
+    entry: object,
+) -> Mapping[str, list[str]] | None:  # mutable-ok: mirrors get_fallback_model_group's fallbacks shape
+    try:
+        validated: Final = _FALLBACK_ENTRY_ADAPTER.validate_python(entry)
+    except ValidationError:
+        return None
+    return {key: _fallback_target_names(targets) for key, targets in validated.items()}
+
+
+def _fallback_entries(configured: object) -> tuple[object, ...]:
+    try:
+        return _FALLBACK_ENTRIES_ADAPTER.validate_python(configured)
+    except ValidationError:
+        return ()
 
 
 def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_group: str | None) -> bool:
@@ -437,9 +455,7 @@ def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_
     if requested_model_group is None or not configured:
         return False
     chains: Final = tuple(
-        _fallback_chain_names(entry)
-        for entry in _FALLBACK_ENTRIES_ADAPTER.validate_python(configured)
-        if isinstance(entry, Mapping)
+        chain for chain in map(_fallback_chain_names, _fallback_entries(configured)) if chain is not None
     )
     fallback_group, _ = get_fallback_model_group(fallbacks=chains, model_group=requested_model_group)
     return bool(fallback_group)

litellm/router_utils/pre_call_checks/model_rate_limit_check.py

@@ -5,15 +5,19 @@ When enabled via router_settings.optional_pre_call_checks: ["enforce_model_rate_
 
 - tpm/rpm: combined TPM + optional RPM (legacy)
 - itpm/otpm: separate input/output tokens per minute
+- tpd/rpd: tokens and requests per UTC calendar day
 
 When a deployment sets both itpm/otpm and tpm/rpm, both are enforced. A warning
 is logged the first time such a deployment is seen.
 """
 
 import contextlib
-from typing import TYPE_CHECKING, Any, Final
+from collections.abc import Callable
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
+from pydantic import BaseModel, ConfigDict
 
 import litellm
 from litellm._internal_context import with_service_target
@@ -33,6 +37,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
     io_token_refund_failure,
 )
 from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
+from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.router import RouterErrors
 from litellm.types.utils import StandardLoggingPayload
 from litellm.utils import get_utc_datetime
@@ -47,6 +52,62 @@ else:
 
 class RoutingArgs:
     ttl: int = 60  # 1min (RPM/TPM expire key)
+    daily_ttl: int = 86400
+
+
+DAY_FORMAT: Final = "%Y-%m-%d"
+
+
+def seconds_until_utc_midnight(now: datetime) -> int:
+    next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
+    return max(1, int((next_midnight - now).total_seconds()))
+
+
+def _first_set(*values: int | None) -> int | None:
+    return next((value for value in values if value is not None), None)
+
+
+class _DailyLimitFields(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+    tpd: int | None = None
+    rpd: int | None = None
+
+
+class _DailyLimitParams(_DailyLimitFields):
+    model: str | None = None
+
+
+class _DailyLimitModelInfo(_DailyLimitFields):
+    id: str | None = None
+
+
+class _DailyLimitDeployment(_DailyLimitFields):
+    model_name: str = ""
+    litellm_params: _DailyLimitParams = _DailyLimitParams()
+    model_info: _DailyLimitModelInfo = _DailyLimitModelInfo()
+
+    def limits(self) -> tuple[int | None, int | None]:
+        return (
+            _first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd),
+            _first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd),
+        )
+
+    def cache_keys(self, current_day: str) -> tuple[str, str]:
+        prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}"
+        return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}"
+
+    def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError:
+        return litellm.RateLimitError(
+            message=f"Model rate limit exceeded. {kind.upper()} limit={limit}, current usage={usage}",
+            llm_provider="",
+            model=self.litellm_params.model,
+            response=httpx.Response(
+                status_code=429,
+                content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}",
+                headers={"retry-after": str(seconds_until_utc_midnight(now))},
+                request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"),
+            ),
+        )
 
 
 class ModelRateLimitingCheck(CustomLogger):
@@ -60,12 +121,60 @@ class ModelRateLimitingCheck(CustomLogger):
     this check actively enforces those limits across ALL routing strategies.
     """
 
-    def __init__(self, dual_cache: DualCache):
+    def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime):
         self.dual_cache = dual_cache
+        self.clock = clock
         # model_ids already warned about itpm/otpm + tpm/rpm on the same deployment,
         # so the warning is logged once per deployment rather than per request.
         self._io_token_conflict_warned_ids: set[str] = set()
 
+    def _increment(self, key: str, value: int, ttl: int) -> int:
+        return self.dual_cache.increment_cache(key=key, value=value, ttl=ttl)
+
+    async def _async_increment(
+        self, key: str, value: int, ttl: int, parent_otel_span: Span | None = None
+    ) -> int | None:
+        count: Final = await self.dual_cache.async_increment_cache(
+            key=key, value=value, ttl=ttl, parent_otel_span=parent_otel_span
+        )
+        return None if count is None else int(count)
+
+    @staticmethod
+    def _request_count_operations(
+        rpm_key: str | None, rpd_key: str | None, increment_value: int
+    ) -> tuple[RedisPipelineIncrementOperation, ...]:
+        return tuple(
+            RedisPipelineIncrementOperation(key=key, increment_value=increment_value, ttl=ttl)
+            for key, ttl in ((rpm_key, RoutingArgs.ttl), (rpd_key, RoutingArgs.daily_ttl))
+            if key is not None
+        )
+
+    async def _async_refund_request_counts(
+        self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None
+    ) -> None:
+        operations: Final = self._request_count_operations(rpm_key, rpd_key, -1)
+        if operations:
+            await self.dual_cache.async_increment_cache_pipeline(
+                increment_list=list(operations), parent_otel_span=parent_otel_span
+            )
+
+    async def _async_increment_request_counts(
+        self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None
+    ) -> tuple[int | None, int | None]:
+        operations: Final = self._request_count_operations(rpm_key, rpd_key, 1)
+        if not operations:
+            return None, None
+        counts: Final = await self.dual_cache.async_increment_cache_pipeline(
+            increment_list=list(operations), parent_otel_span=parent_otel_span
+        )
+        if counts is None or len(counts) != len(operations):
+            return None, None
+        count_by_key: Final = {op["key"]: int(count) for op, count in zip(operations, counts)}
+        return (
+            None if rpm_key is None else count_by_key.get(rpm_key),
+            None if rpd_key is None else count_by_key.get(rpd_key),
+        )
+
     def _warn_io_token_and_tpm_rpm_coexist_once(self, deployment: dict) -> None:
         tpm_limit, rpm_limit = self._get_deployment_limits(deployment)
         if tpm_limit is None and rpm_limit is None:
@@ -129,6 +238,21 @@ class ModelRateLimitingCheck(CustomLogger):
 
         return tpm, rpm
 
+    def _check_daily_limits(self, deployment: _DailyLimitDeployment, now: datetime) -> None:
+        tpd_limit, rpd_limit = deployment.limits()
+        if tpd_limit is None and rpd_limit is None:
+            return
+        tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT))
+        if tpd_limit is not None:
+            current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit)
+            if current_tpd is not None and current_tpd >= tpd_limit:
+                raise deployment.limit_error("tpd", tpd_limit, current_tpd, now)
+        if rpd_limit is not None:
+            current_rpd: Final = self._increment(rpd_key, 1, RoutingArgs.daily_ttl)
+            if current_rpd > rpd_limit:
+                self._increment(rpd_key, -1, RoutingArgs.daily_ttl)
+                raise deployment.limit_error("rpd", rpd_limit, current_rpd, now)
+
     def _get_cache_keys(self, deployment: dict, current_minute: str) -> tuple[str, str]:
         """Get the cache keys for TPM and RPM tracking."""
         model_id: Final = deployment.get("model_info", {}).get("id")
@@ -178,13 +302,10 @@ class ModelRateLimitingCheck(CustomLogger):
                 )
                 io_reservation_made = True
 
+            dt: Final = self.clock()
+            daily: Final = _DailyLimitDeployment.model_validate(deployment)
             tpm_limit, rpm_limit = self._get_deployment_limits(deployment)
 
-            # If no limits are set, allow the request
-            if tpm_limit is None and rpm_limit is None:
-                return deployment
-
-            dt: Final = get_utc_datetime()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute)
 
@@ -213,8 +334,9 @@ class ModelRateLimitingCheck(CustomLogger):
 
             # Check RPM limit (atomic increment-first to avoid race conditions)
             if rpm_limit is not None:
-                current_rpm: Final = self.dual_cache.increment_cache(key=rpm_key, value=1, ttl=RoutingArgs.ttl)
+                current_rpm: Final = self._increment(rpm_key, 1, RoutingArgs.ttl)
                 if current_rpm is not None and current_rpm > rpm_limit:
+                    self._increment(rpm_key, -1, RoutingArgs.ttl)
                     raise litellm.RateLimitError(
                         message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}",
                         llm_provider="",
@@ -230,6 +352,12 @@ class ModelRateLimitingCheck(CustomLogger):
                         ),
                     )
 
+            try:
+                self._check_daily_limits(daily, dt)
+            except litellm.RateLimitError:
+                if rpm_limit is not None:
+                    self._increment(rpm_key, -1, RoutingArgs.ttl)
+                raise
             return deployment
 
         except litellm.RateLimitError:
@@ -259,15 +387,14 @@ class ModelRateLimitingCheck(CustomLogger):
                 )
                 io_reservation_made = True
 
+            dt: Final = self.clock()
+            daily: Final = _DailyLimitDeployment.model_validate(deployment)
+            tpd_limit, rpd_limit = daily.limits()
             tpm_limit, rpm_limit = self._get_deployment_limits(deployment)
 
-            # If no limits are set, allow the request
-            if tpm_limit is None and rpm_limit is None:
-                return deployment
-
-            dt: Final = get_utc_datetime()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute)
+            tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT))
 
             model_id: Final = deployment.get("model_info", {}).get("id")
             model_name: Final = deployment.get("litellm_params", {}).get("model")
@@ -293,30 +420,37 @@ class ModelRateLimitingCheck(CustomLogger):
                         num_retries=0,  # Don't retry - return 429 immediately
                     )
 
-            # Check RPM limit (atomic increment-first to avoid race conditions)
-            if rpm_limit is not None:
-                current_rpm: Final = await self.dual_cache.async_increment_cache(
-                    key=rpm_key,
-                    value=1,
-                    ttl=RoutingArgs.ttl,
-                    parent_otel_span=parent_otel_span,
-                )
-                if current_rpm is not None and current_rpm > rpm_limit:
-                    raise litellm.RateLimitError(
-                        message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}",
-                        llm_provider="",
-                        model=model_name,
-                        response=httpx.Response(
-                            status_code=429,
-                            content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}",
-                            headers={"retry-after": str(60)},
-                            request=httpx.Request(
-                                method="model_rate_limit_check",
-                                url="https://github.com/BerriAI/litellm",
-                            ),
+            if tpd_limit is not None:
+                current_tpd: Final = await self._async_get_current_tpm(tpd_key, tpd_limit, parent_otel_span)
+                if current_tpd is not None and current_tpd >= tpd_limit:
+                    raise daily.limit_error("tpd", tpd_limit, current_tpd, dt)
+
+            charged_rpm_key: Final = rpm_key if rpm_limit is not None else None
+            charged_rpd_key: Final = rpd_key if rpd_limit is not None else None
+            current_rpm, current_rpd = await self._async_increment_request_counts(
+                charged_rpm_key, charged_rpd_key, parent_otel_span
+            )
+            if rpm_limit is not None and current_rpm is not None and current_rpm > rpm_limit:
+                await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span)
+                raise litellm.RateLimitError(
+                    message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}",
+                    llm_provider="",
+                    model=model_name,
+                    response=httpx.Response(
+                        status_code=429,
+                        content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}",
+                        headers={"retry-after": str(60)},
+                        request=httpx.Request(
+                            method="model_rate_limit_check",
+                            url="https://github.com/BerriAI/litellm",
                         ),
-                        num_retries=0,  # Don't retry - return 429 immediately
-                    )
+                    ),
+                    num_retries=0,  # Don't retry - return 429 immediately
+                )
+
+            if rpd_limit is not None and current_rpd is not None and current_rpd > rpd_limit:
+                await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span)
+                raise daily.limit_error("rpd", rpd_limit, current_rpd, dt)
 
             return deployment
 
@@ -373,17 +507,16 @@ class ModelRateLimitingCheck(CustomLogger):
             if not model or not total_tokens:
                 return
 
-            dt: Final = get_utc_datetime()
+            dt: Final = self.clock()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
+            tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
 
             verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens)
 
-            await self.dual_cache.async_increment_cache(
-                key=tpm_key,
-                value=total_tokens,
-                ttl=RoutingArgs.ttl,
-            )
+            tokens: Final = int(total_tokens)
+            await self._async_increment(tpm_key, tokens, RoutingArgs.ttl)
+            await self._async_increment(tpd_key, tokens, RoutingArgs.daily_ttl)
 
         except Exception as e:
             verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_log_success_event: %s", e)
@@ -434,15 +567,14 @@ class ModelRateLimitingCheck(CustomLogger):
             if not model or not total_tokens:
                 return
 
-            dt: Final = get_utc_datetime()
+            dt: Final = self.clock()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
+            tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
 
-            self.dual_cache.increment_cache(
-                key=tpm_key,
-                value=total_tokens,
-                ttl=RoutingArgs.ttl,
-            )
+            tokens: Final = int(total_tokens)
+            self._increment(tpm_key, tokens, RoutingArgs.ttl)
+            self._increment(tpd_key, tokens, RoutingArgs.daily_ttl)
 
         except Exception as e:
             verbose_router_logger.debug("Error in ModelRateLimitingCheck.log_success_event: %s", e)

litellm/router_utils/request_priority.py

@@ -47,6 +47,15 @@ def resolve_request_priority(
     return priority
 
 
+def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, int | str]:
+    if not header:
+        return {}
+    priority: Final = _as_priority(header)
+    if isinstance(priority, InvalidPriority):
+        return {} if drops_params() else {"priority": header}
+    return {"priority": priority}
+
+
 def parse_default_priority(value: object) -> int | None | InvalidPriority:
     if value is None:
         return None