Merge remote-tracking branch 'origin/main' into litellm_internal_copy_40078

# Conflicts:
#	litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

.agents/skills/database-access/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: database-access
 description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
+metadata:
+  internal: true
 ---
 
 # Database access

.github/workflows/spend-logs-guard.yml

@@ -4,10 +4,6 @@ on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the P
   pull_request_target:
     branches:
       - main
-  pull_request_review:
-    types:
-      - submitted
-      - dismissed
 
 permissions:
   contents: read

.github/workflows/test-unit.yml

@@ -137,6 +137,7 @@ jobs:
         include:
           - shard: core-utils
             test-path: |-
+              tests/unit/accounting
               tests/unit/batch_completion
               tests/unit/decisions
               tests/unit/extraction

ci_cd/spend_logs_guard.py

@@ -2,9 +2,12 @@
 needs an approving review from the spend-log owner before it can merge.
 
 CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
-so a path rule would put every migration on the owner. This gate reads the diff instead: when no
-added line in the watched files mentions the table it passes at once, otherwise it passes only once
-the owner's latest review on the pull request is an approval. The workflow re-runs on every review.
+so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma
+hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration
+hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes
+only once the owner's latest review on the pull request is an approval that still covers every spend-log line
+in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw).
+Reviews do not trigger the workflow, so the check is re-run by hand after approving.
 """
 
 from __future__ import annotations
@@ -13,8 +16,10 @@ import argparse
 import re
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
 from pathlib import Path
+from types import MappingProxyType
 from typing import Final
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter
@@ -25,7 +30,10 @@ WATCHED: Final = (
     "litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
 )
 MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
-TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs", re.IGNORECASE)
+TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
+DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
+MODEL_START: Final = re.compile(r"^model\s+LiteLLM_SpendLogs\s*\{")
+HUNK: Final = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@[^\n]*\n", re.MULTILINE)
 
 
 class Reviewer(BaseModel):
@@ -37,31 +45,77 @@ class Review(BaseModel):
     model_config = ConfigDict(frozen=True, extra="ignore")
     state: str
     user: Reviewer
+    commit_id: str | None = None
 
 
 _REVIEWS: Final = TypeAdapter(list[Review])
 
 
+@dataclass(frozen=True, slots=True)
+class Block:
+    """Lines of `model LiteLLM_SpendLogs { ... }` in one schema.prisma copy at the merge base and at the head."""
+
+    base: range
+    head: range
+
+
+@dataclass(frozen=True, slots=True)
+class Hunk:
+    removed: tuple[tuple[int, str], ...]
+    added: tuple[tuple[int, str], ...]
+
+
 def watched(path: str) -> bool:
     return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
 
 
-def _added_lines(section: str) -> tuple[str, ...]:
-    body: Final = section.split("\n@@", 1)
-    return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
+def model_block(schema: str) -> range:
+    """1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none."""
+    lines: Final = schema.splitlines()
+    starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line))
+    if not starts:
+        return range(0)
+    ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}"))
+    return range(starts[0], (ends[0] if ends else len(lines)) + 1)
 
 
-def spend_log_hits(diff: str) -> tuple[str, ...]:
-    """`path: added line` for every added line in a watched file that names the table."""
+def _hunks(section: str) -> tuple[Hunk, ...]:
+    headers: Final = tuple(HUNK.finditer(section))
+    ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),)
+    return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends))
+
+
+def _hunk(header: re.Match[str], body: str) -> Hunk:
+    lines: Final = body.splitlines()
+    return Hunk(
+        removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))),
+        added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))),
+    )
+
+
+def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    """`path: +line` or `path: -line` for every changed line that touches the table.
+
+    A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines,
+    merge base for removed ones), a migration line when it is added and names the table."""
     sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
-    return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
-
-
-def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
-    pairs: Final = tuple((_path_of(section), section) for section in sections)
+    paths: Final = tuple((_path_of(section), section) for section in sections)
     return tuple(
-        (path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
-    )  # comprehension-ok: path then its lines
+        hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks)
+    )  # comprehension-ok: path then its hits
+
+
+def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    if path not in blocks:
+        return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line))
+    block: Final = blocks[path]
+    removed: Final = tuple(
+        f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base
+    )  # comprehension-ok: hunk then its lines
+    added: Final = tuple(
+        f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head
+    )  # comprehension-ok: hunk then its lines
+    return removed + added
 
 
 def _path_of(section: str) -> str:
@@ -70,11 +124,33 @@ def _path_of(section: str) -> str:
 
 
 def approved_by(reviews: Sequence[Review], login: str) -> bool:
-    """True when `login`'s most recent review on the pull request is an approval."""
-    states: Final = tuple(review.state for review in reviews if review.user.login == login)
+    """True when `login`'s most recent review that changes GitHub's approval state is an approval.
+
+    COMMENTED and PENDING reviews leave an earlier approval in force on GitHub, so they are skipped here too.
+    """
+    own: Final = tuple(review.state for review in reviews if review.user.login == login)
+    states: Final = tuple(state for state in own if state in DECIDING_STATES)
     return bool(states) and states[-1] == "APPROVED"
 
 
+def latest_approval(reviews: Sequence[Review], login: str) -> Review | None:
+    """`login`'s most recent APPROVED review on the pull request, the one GitHub keeps in force after pushes."""
+    own: Final = tuple(review for review in reviews if review.user.login == login and review.state == "APPROVED")
+    return own[-1] if own else None
+
+
+def approval_covers(
+    approval: Review, head: str, hits: Sequence[str], hits_at: Callable[[str], Sequence[str] | None]
+) -> bool:
+    """True when the approval is on the head, or the head adds no spend-log line the approved commit did not."""
+    if approval.commit_id is None:
+        return False
+    if approval.commit_id == head:
+        return True
+    reviewed: Final = hits_at(approval.commit_id)
+    return reviewed is not None and set(hits) <= set(reviewed)
+
+
 def added_diff(base: str, head: str) -> str:
     result: Final = subprocess.run(
         ("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX),
@@ -85,6 +161,27 @@ def added_diff(base: str, head: str) -> str:
     return result.stdout
 
 
+def file_at(revision: str, path: str) -> str:
+    """Contents of `path` at `revision`, empty when the file does not exist there."""
+    result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True)
+    return result.stdout if result.returncode == 0 else ""
+
+
+def schema_blocks(base: str, head: str) -> Mapping[str, Block]:
+    return MappingProxyType(
+        {path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED}
+    )
+
+
+def fetched(commit: str) -> bool:
+    result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
+    return result.returncode == 0
+
+
+def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
+    return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None
+
+
 class Arguments(BaseModel):
     model_config = ConfigDict(frozen=True)
     base: str
@@ -100,18 +197,28 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
     parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
     args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
-    hits: Final = spend_log_hits(added_diff(args.base, args.head))
+    hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head))
     if not hits:
-        sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
+        sys.stdout.write(
+            "No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; "
+            "nothing to review.\n"
+        )
         return 0
-    if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver):
+    reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
+    approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None
+    if approval is not None and approval_covers(
+        approval, args.head, hits, lambda commit: reviewed_hits(args.base, commit)
+    ):
         sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n")
         return 0
     listed: Final = "".join(f"  {hit}\n" for hit in hits)
-    sys.stdout.write(
+    reason: Final = (
         f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n"
-        f"{listed}The check re-runs when a review is submitted.\n"
+        if approval is None
+        else f"LiteLLM_SpendLogs lines changed after @{args.approver} approved {approval.commit_id}, "
+        f"a new approving review is needed:\n"
     )
+    sys.stdout.write(f"{reason}{listed}After approving, re-run this job.\n")
     return 1
 
 

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -32,6 +32,7 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.llms.base_llm.guardrail_translation.utils import effective_skip_tool_message_for_guardrail
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails._content_utils import iter_tool_json_fields, map_tool_json_text, walk_user_text
 from litellm.types.guardrails import GuardrailEventHooks
@@ -753,7 +754,12 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         _masked_entity_count.set({})
 
         # Covers multimodal list content + Responses-API input.
-        inspected = walk_user_text(data, self.redact_text, self.redact_tool_payload) + self._redact_prompt(data)
+        inspected = walk_user_text(
+            data,
+            self.redact_text,
+            self.redact_tool_payload,
+            skip_tool_message=effective_skip_tool_message_for_guardrail(self),
+        ) + self._redact_prompt(data)
 
         if inspected == 0:
             # Image-only, empty-text, and unsupported payloads inspected

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -28,7 +28,7 @@ import httpx
 from fastapi import HTTPException
 from openai import APIConnectionError, AsyncOpenAI
 from openai.types.file_deleted import FileDeleted
-from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, Unpack
 
 import litellm
@@ -53,7 +53,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_metadata,
 )
 from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
-from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
 from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
@@ -309,6 +309,7 @@ def _is_xai_batch_output(
 class _ManagedFileRow(Protocol):
     unified_file_id: str
     file_object: OpenAIFileObject
+    model_mappings: Mapping[str, str]
     flat_model_file_ids: Sequence[str]
     storage_backend: Optional[str]
     storage_url: Optional[str]
@@ -362,6 +363,8 @@ class _ManagedResourceDatabase(Protocol):
     @property
     def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
 
+    async def query_raw(self, query: str, *args: object) -> object: ...
+
     @property
     def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
 
@@ -385,10 +388,83 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
+    return prisma_client.db
+
+
 def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedFileTableActions:
     return prisma_client.db.litellm_managedfiletable
 
 
+class _ListedFileRow(Protocol):
+    @property
+    def unified_file_id(self) -> str: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+    @property
+    def model_mappings(self) -> Mapping[str, object]: ...
+
+    @property
+    def storage_backend(self) -> str | None: ...
+
+
+class _ScopedFileRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    unified_file_id: str
+    file_object: JsonValue = None
+    model_mappings: Mapping[str, JsonValue]
+    storage_backend: str | None = None
+
+
+_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...])
+
+
+def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]:
+    match owner_filter:
+        case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}:
+            return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id)
+        case {"created_by": str() as user_id}:
+            return '"created_by" = $1', (user_id,)
+        case {"team_id": str() as team_id}:
+            return '"team_id" = $1', (team_id,)
+        case _:
+            return ("FALSE", ()) if owner_filter else ("TRUE", ())
+
+
+async def _scoped_file_rows(
+    database: _ManagedResourceDatabase,
+    owner_filter: Mapping[str, object],
+    model_mapping_keys: frozenset[str],
+    cursor_id: str | None,
+    direction: Literal["asc", "desc"],
+    take: int,
+) -> tuple[_ScopedFileRow, ...]:
+    owner_condition, owner_params = _owner_condition(owner_filter)
+    keys_param: Final = len(owner_params) + 1
+    comparison: Final = "<" if direction == "desc" else ">"
+    cursor_condition: Final = (
+        f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" '
+        f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) '
+        if cursor_id
+        else ""
+    )
+    order: Final = direction.upper()
+    rows: Final = await database.query_raw(
+        'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" '
+        f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) '
+        f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} '
+        f"LIMIT ${keys_param + (2 if cursor_id else 1)}",
+        *owner_params,
+        sorted(model_mapping_keys),
+        *((cursor_id,) if cursor_id else ()),
+        take,
+    )
+    return _SCOPED_FILE_ROWS.validate_python(rows)
+
+
 def _iter_provider_file_id_pairs(
     rows: Sequence[_ManagedFileRow],
     requested_provider_file_ids: frozenset[str],
@@ -2265,14 +2341,23 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         user_api_key_dict: UserAPIKeyAuth,
         limit: Optional[int] = None,
         after: Optional[str] = None,
+        model_scope: Optional[FileListModelScope] = None,
+        before: Optional[str] = None,
         **data: Dict,
     ) -> FileListPage:
         """List the managed files the caller owns, newest first.
 
         Pagination is keyset based on ``unified_file_id`` so a key that owns
         every file on the proxy still reads one bounded page at a time.
-        ``purpose`` is applied after parsing, because the managed file table
-        keeps it inside the ``file_object`` blob instead of a column, and rows
+        ``after`` pages toward older files and ``before`` toward newer ones,
+        each page still ordered newest first.
+        ``model_scope`` is applied in the query, so only rows whose
+        ``model_mappings`` name one of its deployment ids or model names, and
+        storage-backed rows, leave the database; a storage-backed row then
+        stays only when its model name is one the router sends to the listed
+        provider. ``purpose`` is applied after parsing, because the managed
+        file table keeps it inside the ``file_object`` blob instead of a
+        column. Rows
         whose blob will not parse drop out there too, so a chunk of rows can
         yield fewer matches than the page holds. Successive chunks are read
         until the page is full or the caller's rows run out, which keeps
@@ -2281,9 +2366,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         scan has to continue past it, the chunk widens to
         ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
         that many rows instead of one per page. That bound is per query, not
-        per request: the work is still linear in the rows the caller owns, and
-        a filter matching nothing reads every one of them, with no index
-        covering either the owner filter or the sort.
+        per request: a ``purpose`` filter matching nothing still reads every
+        row that passes the query.
         """
         validate_file_list_limit(limit)
         validate_file_list_purpose(purpose)
@@ -2292,36 +2376,60 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if owner_filter is None:
             return FileListPage.model_validate(build_list_page([]))
 
-        if after:
-            cursor_row = await _managed_file_table(self.prisma_client).find_first(
-                where={**owner_filter, "unified_file_id": after}
+        if after and before:
+            raise ProxyException(
+                message="Pass either 'after' (or 'after_id') or 'before_id' when listing files, not both.",
+                type="invalid_request_error",
+                param="before_id",
+                code=400,
+                openai_code="invalid_value",
+            )
+        cursor_param: Final = "before_id" if before else "after"
+        cursor: Final = before or after
+        database: Final = _managed_database(self.prisma_client)
+        if cursor:
+            cursor_row = await database.litellm_managedfiletable.find_first(
+                where={**owner_filter, "unified_file_id": cursor}
             )
             if cursor_row is None:
                 raise ProxyException(
-                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    message=f"Invalid '{cursor_param}' cursor: no file found with id '{cursor}'.",
                     type="invalid_request_error",
-                    param="after",
+                    param=cursor_param,
                     code=400,
                     openai_code="invalid_value",
                 )
 
+        if model_scope is not None and not model_scope.model_mapping_keys:
+            return FileListPage.model_validate(build_list_page([]))
+
         page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
         matches: Final[List[OpenAIFileObject]] = []
-        cursor_id = after
+        cursor_id = cursor
         chunk_size = page_size + 1
+        direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc"
 
         while len(matches) <= page_size:
             cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
-            chunk = await _managed_file_table(self.prisma_client).find_many(
-                where=owner_filter,
-                take=chunk_size,
-                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
-                **cursor_args,
+            chunk: Sequence[_ListedFileRow] = (
+                await _scoped_file_rows(
+                    database, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size
+                )
+                if model_scope is not None
+                else await database.litellm_managedfiletable.find_many(
+                    where=owner_filter,
+                    take=chunk_size,
+                    order=[{"created_at": direction}, {"unified_file_id": direction}],
+                    **cursor_args,
+                )
             )
             matches.extend(
                 _public_file_object(parsed_file_object, row.unified_file_id)
                 for row in chunk
-                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                if (
+                    model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
+                )
+                and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
                 and (purpose is None or parsed_file_object.purpose == purpose)
             )
             if len(chunk) < chunk_size:
@@ -2329,7 +2437,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             cursor_id = chunk[-1].unified_file_id
             chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
 
-        return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
+        page: Final = matches[:page_size]
+        return FileListPage.model_validate(
+            build_list_page(page[::-1] if before else page, has_more=len(matches) > page_size)
+        )
 
     def _is_batch_polling_enabled(self) -> bool:
         """

litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py

@@ -343,11 +343,16 @@ def btree_max_entry_bytes(block_size: int) -> int:
 
 
 def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed":
+    """The bytes a value takes in the index entry: its length, or its stored size when
+    Postgres compressed it, since `index_form_tuple` keeps the compressed bytes."""
     from psycopg import sql
 
-    if column.type_name in _OCTET_LENGTH_TYPES:
-        return sql.SQL("octet_length({})").format(identifier)
-    return sql.SQL("octet_length({}::text)").format(identifier)
+    length: Final = (
+        sql.SQL("octet_length({})").format(identifier)
+        if column.type_name in _OCTET_LENGTH_TYPES
+        else sql.SQL("octet_length({}::text)").format(identifier)
+    )
+    return sql.SQL("least({}, pg_column_size({}))").format(length, identifier)
 
 
 def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed":

litellm-proxy-extras/litellm_proxy_extras/migrations/20261021000000_managed_file_created_by_created_at_index/migration.sql

@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_ManagedFileTable_created_by_created_at_idx"
+ON "LiteLLM_ManagedFileTable" ("created_by", "created_at" DESC);

litellm-proxy-extras/litellm_proxy_extras/migrations/20261027000000_add_accounting_ledger_tables/migration.sql

@@ -0,0 +1,47 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE TYPE "AccountingEventKind" AS ENUM ('inbound_usage', 'outbound_usage', 'cost');
+
+CREATE TYPE "AccountingDeliveryState" AS ENUM ('pending', 'applied', 'parked');
+
+CREATE TYPE "AccountingDeliveryTarget" AS ENUM ('LiteLLM_VerificationToken', 'LiteLLM_UserTable', 'LiteLLM_TeamTable', 'LiteLLM_TeamMembership', 'LiteLLM_OrganizationTable', 'LiteLLM_OrganizationMembership', 'LiteLLM_EndUserTable', 'LiteLLM_AgentsTable', 'LiteLLM_TagTable', 'LiteLLM_ProjectTable', 'LiteLLM_ModelAccessGroupBudgetTable', 'LiteLLM_BudgetTable', 'LiteLLM_BudgetWindowSpend', 'LiteLLM_DailyUserSpend', 'LiteLLM_DailyTeamSpend', 'LiteLLM_DailyOrganizationSpend', 'LiteLLM_DailyEndUserSpend', 'LiteLLM_DailyAgentSpend', 'LiteLLM_DailyTagSpend', 'LiteLLM_DailyModelUsage', 'LiteLLM_DailyToolSpend', 'LiteLLM_AutoRouterDailySpend');
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingEvent" (
+    "event_id" TEXT NOT NULL,
+    "kind" "AccountingEventKind" NOT NULL,
+    "source_event_id" TEXT,
+    "schema_version" INTEGER NOT NULL,
+    "payload_hash" TEXT NOT NULL,
+    "occurred_at" TIMESTAMP(3) NOT NULL,
+    "recorded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "payload" JSONB NOT NULL,
+
+    CONSTRAINT "LiteLLM_AccountingEvent_schema_version_check" CHECK ("schema_version" > 0),
+    CONSTRAINT "LiteLLM_AccountingEvent_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingDelivery" (
+    "event_id" TEXT NOT NULL,
+    "target" "AccountingDeliveryTarget" NOT NULL,
+    "version" INTEGER NOT NULL,
+    "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "applied_at" TIMESTAMP(3),
+    "last_error" TEXT,
+
+    CONSTRAINT "LiteLLM_AccountingDelivery_version_check" CHECK ("version" > 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_attempts_check" CHECK ("attempts" >= 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_pkey" PRIMARY KEY ("event_id","target","version")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_recorded_at_idx" ON "LiteLLM_AccountingEvent"("recorded_at");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_source_event_id_idx" ON "LiteLLM_AccountingEvent"("kind", "source_event_id");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingDelivery_target_state_next_attempt_at_idx" ON "LiteLLM_AccountingDelivery"("target", "state", "next_attempt_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
   @@index([unified_file_id])
   @@index([flat_model_file_ids], type: Gin)
   @@index([team_id, created_at(sort: Desc)])
+  @@index([created_by, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
   @@index([window_end])
   @@index([created_at])
 }
+
+enum AccountingEventKind {
+  inbound_usage
+  outbound_usage
+  cost
+}
+
+enum AccountingDeliveryState {
+  pending
+  applied
+  parked
+}
+
+enum AccountingDeliveryTarget {
+  LiteLLM_VerificationToken
+  LiteLLM_UserTable
+  LiteLLM_TeamTable
+  LiteLLM_TeamMembership
+  LiteLLM_OrganizationTable
+  LiteLLM_OrganizationMembership
+  LiteLLM_EndUserTable
+  LiteLLM_AgentsTable
+  LiteLLM_TagTable
+  LiteLLM_ProjectTable
+  LiteLLM_ModelAccessGroupBudgetTable
+  LiteLLM_BudgetTable
+  LiteLLM_BudgetWindowSpend
+  LiteLLM_DailyUserSpend
+  LiteLLM_DailyTeamSpend
+  LiteLLM_DailyOrganizationSpend
+  LiteLLM_DailyEndUserSpend
+  LiteLLM_DailyAgentSpend
+  LiteLLM_DailyTagSpend
+  LiteLLM_DailyModelUsage
+  LiteLLM_DailyToolSpend
+  LiteLLM_AutoRouterDailySpend
+}
+
+// Durable accounting ledger. One row per usage or cost event; payload holds the
+// private event envelope versioned by schema_version. Usage rows retain request-scoped
+// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
+model LiteLLM_AccountingEvent {
+  event_id        String              @id
+  kind            AccountingEventKind
+  source_event_id String?
+  schema_version  Int
+  payload_hash    String
+  occurred_at     DateTime
+  recorded_at     DateTime            @default(now())
+  payload         Json
+
+  @@index([recorded_at])
+  @@index([kind, source_event_id])
+}
+
+// One row per (event, target table, version): pending work for the projector and,
+// once applied in the same transaction as the target write, the receipt that stops a retry.
+model LiteLLM_AccountingDelivery {
+  event_id        String
+  target          AccountingDeliveryTarget
+  version         Int
+  state           AccountingDeliveryState  @default(pending)
+  attempts        Int                      @default(0)
+  next_attempt_at DateTime                 @default(now())
+  applied_at      DateTime?
+  last_error      String?
+
+  @@id([event_id, target, version])
+  @@index([target, state, next_attempt_at])
+}

litellm-rust/.agents/skills/rust-error/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-error
 description: Define, wrap, convert, classify, report or test Rust error types in litellm-rust with thiserror. Use when adding an error type or variant, mapping one crate's error into another, turning an error into a gateway response or Python exception, deciding retry or fault from an error, or replacing call sites that build errors from strings
+metadata:
+  internal: true
 ---
 
 # Rust errors

litellm-rust/.agents/skills/rust-gateway/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-gateway
 description: Add or change an HTTP route, extractor, response envelope, stream or router mount in the litellm-rust gateway crates (crates/gateway and crates/gateway-*). Use when adding an endpoint, parsing a request body, choosing an error envelope for an API surface, relaying SSE, or writing a router-level test
+metadata:
+  internal: true
 ---
 
 # Rust gateway

litellm-rust/.agents/skills/rust-string-enums/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-string-enums
 description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas
+metadata:
+  internal: true
 ---
 
 # Rust string enums

litellm-rust/.agents/skills/rust-tracing/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-tracing
 description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery
+metadata:
+  internal: true
 ---
 
 # Rust tracing

litellm-rust/crates/core-utils/src/core_helpers.rs

@@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
     ("max_tokens", "length"),
     ("refusal", "content_filter"),
     ("compaction", "length"),
+    ("model_context_window_exceeded", "length"),
     ("guardrail_intervened", "content_filter"),
     ("content_filtered", "content_filter"),
     ("content_filter", "content_filter"),
@@ -74,6 +75,7 @@ mod tests {
     #[case::end_turn("end_turn", "stop")]
     #[case::stop_sequence("stop_sequence", "stop")]
     #[case::max_tokens("max_tokens", "length")]
+    #[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
     #[case::refusal("refusal", "content_filter")]
     #[case::guardrail_intervened("guardrail_intervened", "content_filter")]
     #[case::content_filtered("content_filtered", "content_filter")]

litellm/accounting/__init__.py

@@ -0,0 +1,62 @@
+"""Public accounting contracts. Importing these values does not enable accounting."""
+
+from litellm.accounting.measurements import measurements_for
+from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter
+from litellm.accounting.types import (
+    AdmissionDecision,
+    AggregateQuery,
+    AggregateRow,
+    CostEvent,
+    DeliveryState,
+    DeliveryTarget,
+    Dimension,
+    DimensionFilter,
+    DimensionName,
+    EventKind,
+    Granularity,
+    InboundUsageEvent,
+    Measurement,
+    Metric,
+    MetricName,
+    Multiplier,
+    NamedCost,
+    OutboundUsageEvent,
+    Outcome,
+    PricedQuantity,
+    PricingSnapshot,
+    Rate,
+    Unpriced,
+    validate_aggregate_rows,
+)
+
+__all__ = [
+    "AdmissionDecision",
+    "AggregateQuery",
+    "AggregateRow",
+    "BudgetEnforcer",
+    "CostAggregator",
+    "CostCompiler",
+    "CostEvent",
+    "DeliveryState",
+    "DeliveryTarget",
+    "Dimension",
+    "DimensionFilter",
+    "DimensionName",
+    "EventKind",
+    "Granularity",
+    "InboundUsageEvent",
+    "Measurement",
+    "Metric",
+    "MetricName",
+    "Multiplier",
+    "NamedCost",
+    "OutboundUsageEvent",
+    "Outcome",
+    "PricedQuantity",
+    "PricingSnapshot",
+    "Rate",
+    "Unpriced",
+    "UsageMeter",
+    "measurements_for",
+    "validate_aggregate_rows",
+]

litellm/accounting/measurements.py

@@ -0,0 +1,13 @@
+from typing import Final
+
+from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent
+
+
+def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]:
+    """Return the complete contribution set for one event, ready for one apply call."""
+    metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics
+    occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at
+    return tuple(
+        Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric)
+        for metric in metrics or ()
+    )

litellm/accounting/protocols.py

@@ -0,0 +1,85 @@
+"""Contracts only. Implementations own persistence, pricing, projections and live budgets.
+
+The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
+Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
+model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
+These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
+Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
+T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
+"""
+
+from typing import Protocol
+
+from litellm.accounting.types import (
+    AdmissionDecision,
+    AggregateQuery,
+    AggregateRow,
+    CostEvent,
+    Dimension,
+    InboundUsageEvent,
+    Measurement,
+    Metric,
+    OutboundUsageEvent,
+    Unpriced,
+)
+
+
+class UsageMeter(Protocol):
+    async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
+        """Acknowledge commit, no-op identical IDs and reject conflicting content.
+
+        Persist the event and its request-scoped pricing context before acknowledging.
+        HTTP acknowledgement is a separate policy. Unknown usage remains None.
+        """
+        ...
+
+
+class CostCompiler(Protocol):
+    async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
+        """Reuse the committed charge or finalize one charge and its delivery intent.
+
+        Resolve captured request-scoped context by event identity and cost-map rates
+        at compilation. Persist the resolved inputs in the required snapshot. Replay
+        reuses the committed result and never consumes free allowance again.
+        Missing pricing is Unpriced, never an inferred zero.
+        """
+        ...
+
+
+class CostAggregator(Protocol):
+    async def apply(self, measurements: tuple[Measurement, ...]) -> None:
+        """Apply one source event's complete contribution set per target/version.
+
+        Each target commits all contributions and its receipt atomically. A receipt
+        for the first metric must not suppress the rest. Usage need not wait for cost.
+        """
+        ...
+
+    async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
+        """Return disjoint [start, end) rows per group using existing daily projections.
+
+        Filters combine with AND, values within a filter with IN. DAY emits daily
+        buckets; None emits whole-interval totals. Reject unsupported metric/scope
+        combinations. Different groups may overlap, including non-additive tags.
+        """
+        ...
+
+
+class BudgetEnforcer(Protocol):
+    async def reserve(
+        self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...]
+    ) -> AdmissionDecision:
+        """Apply existing admission policies and estimates; duplicate IDs add no hold."""
+        ...
+
+    async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None:
+        """Settle the committed charge with atomic deduplication and hold mutation.
+
+        Preserve scope/window attribution and retry unfinished settlement after
+        restart. SQL projection receipts do not provide live-counter atomicity.
+        """
+        ...
+
+    async def release(self, reservation_id: str) -> None:
+        """Idempotently release unused exposure at the request boundary, preserving charges."""
+        ...

litellm/accounting/types.py

@@ -0,0 +1,428 @@
+"""Accounting values use UTC instants, decimal strings on the wire and USD charges."""
+
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from decimal import Decimal
+from enum import Enum
+from typing import Annotated, Final, Literal, TypeAlias
+
+from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator
+from typing_extensions import Self
+
+from litellm.types.utils import CallTypes
+
+_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True)
+Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")]
+Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)]
+
+
+def _utc(value: datetime) -> datetime:
+    return value.astimezone(timezone.utc)
+
+
+Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)]
+
+
+class MetricName(str, Enum):
+    """Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens."""
+
+    REQUESTS = "requests"
+    SUCCESSFUL_REQUESTS = "successful_requests"
+    FAILED_REQUESTS = "failed_requests"
+    PROMPT_TOKENS = "prompt_tokens"
+    COMPLETION_TOKENS = "completion_tokens"
+    REASONING_TOKENS = "reasoning_tokens"
+    TOTAL_TOKENS = "total_tokens"
+    CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens"
+    CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens"
+    CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr"
+    SPEND = "spend"
+
+
+class DimensionName(str, Enum):
+    KEY = "key"
+    USER = "user"
+    TEAM = "team"
+    ORGANIZATION = "organization"
+    END_USER = "end_user"
+    MODEL = "model"
+    MODEL_GROUP = "model_group"
+    DEPLOYMENT = "deployment"
+    PROVIDER = "provider"
+    TOOL = "tool"
+    AGENT = "agent"
+    PROJECT = "project"
+    MODEL_ACCESS_GROUP = "model_access_group"
+    API_BASE = "api_base"
+    SERVICE_TIER = "service_tier"
+    TAG = "tag"
+
+
+class Outcome(str, Enum):
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+class Granularity(str, Enum):
+    DAY = "day"
+
+
+class EventKind(str, Enum):
+    INBOUND_USAGE = "inbound_usage"
+    OUTBOUND_USAGE = "outbound_usage"
+    COST = "cost"
+
+
+class DeliveryState(str, Enum):
+    PENDING = "pending"
+    APPLIED = "applied"
+    PARKED = "parked"
+
+
+class DeliveryTarget(str, Enum):
+    """One member per existing table the projector writes; each is applied and receipted independently."""
+
+    KEY_SPEND = "LiteLLM_VerificationToken"
+    USER_SPEND = "LiteLLM_UserTable"
+    TEAM_SPEND = "LiteLLM_TeamTable"
+    TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership"
+    ORGANIZATION_SPEND = "LiteLLM_OrganizationTable"
+    ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership"
+    END_USER_SPEND = "LiteLLM_EndUserTable"
+    AGENT_SPEND = "LiteLLM_AgentsTable"
+    TAG_SPEND = "LiteLLM_TagTable"
+    PROJECT_SPEND = "LiteLLM_ProjectTable"
+    MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable"
+    BUDGET_SPEND = "LiteLLM_BudgetTable"
+    BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend"
+    DAILY_USER_SPEND = "LiteLLM_DailyUserSpend"
+    DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend"
+    DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend"
+    DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend"
+    DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend"
+    DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend"
+    DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage"
+    DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend"
+    AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend"
+
+
+_SUBSET_PAIRS: Final = (
+    (MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS),
+    (MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS),
+)
+
+
+def _check_subsets(values: Mapping[str, Decimal]) -> None:
+    for part, whole in _SUBSET_PAIRS:
+        if part.value in values and whole.value in values and values[part.value] > values[whole.value]:
+            raise ValueError(f"{part.value} cannot exceed {whole.value}")
+
+
+class Metric(BaseModel):
+    model_config = _CONFIG
+
+    name: MetricName
+    value: Quantity
+    unit: Literal["count", "USD"] = "count"
+
+    @model_validator(mode="after")
+    def validate_unit(self) -> Self:
+        if (self.name is MetricName.SPEND) != (self.unit == "USD"):
+            raise ValueError("SPEND requires USD; usage metrics require count")
+        if self.unit == "count" and self.value != self.value.to_integral_value():
+            raise ValueError("Count metrics require integral values")
+        return self
+
+
+class Dimension(BaseModel):
+    model_config = _CONFIG
+
+    name: DimensionName
+    value: Identifier
+
+
+_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP})
+
+
+def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]:
+    if len(frozenset(values)) != len(values):
+        raise ValueError("Duplicate dimension values")
+    single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS)
+    if len(frozenset(single_names)) != len(single_names):
+        raise ValueError("Only tag and model access group dimensions may have multiple values")
+    return values
+
+
+Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)]
+_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS))
+
+
+def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]:
+    if len(frozenset(value.name for value in values)) != len(values):
+        raise ValueError("Duplicate metric names")
+    return values
+
+
+Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)]
+
+
+def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None:
+    if end < start or (end == start and not allow_empty):
+        raise ValueError("End must follow start")
+
+
+class InboundUsageEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    request_id: Identifier
+    call_type: CallTypes
+    started_at: Instant
+    ended_at: Instant
+    outcome: Outcome
+    dimensions: Dimensions
+    metrics: Metrics
+
+    @model_validator(mode="after")
+    def validate_caller_activity(self) -> Self:
+        _interval(self.started_at, self.ended_at, allow_empty=True)
+        if any(metric.name not in _CALLER_METRICS for metric in self.metrics):
+            raise ValueError("Inbound events own only caller request metrics")
+        if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics):
+            raise ValueError("Inbound events count exactly one caller request")
+        for metric in self.metrics:
+            if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED):
+                raise ValueError("Successful request count must match outcome")
+            if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED):
+                raise ValueError("Failed request count must match outcome")
+        return self
+
+
+class OutboundUsageEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    attempt_id: Identifier
+    request_id: Identifier | None
+    retry_of_attempt_id: Identifier | None
+    call_type: CallTypes
+    started_at: Instant
+    ended_at: Instant
+    outcome: Outcome
+    dimensions: Dimensions
+    metrics: Metrics | None
+
+    @model_validator(mode="after")
+    def validate_attempt_usage(self) -> Self:
+        _interval(self.started_at, self.ended_at, allow_empty=True)
+        if self.retry_of_attempt_id == self.attempt_id:
+            raise ValueError("An attempt cannot retry itself")
+        if self.metrics is not None and any(
+            metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics
+        ):
+            raise ValueError("Outbound events own only attempt usage metrics")
+        if self.metrics is not None:
+            _check_subsets({metric.name.value: metric.value for metric in self.metrics})
+        return self
+
+
+PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"]
+
+
+class Rate(BaseModel):
+    model_config = _CONFIG
+
+    measure: Identifier
+    unit: PricingUnit
+    usd_per_unit: Quantity
+
+
+class PricedQuantity(BaseModel):
+    """A pricing input in its billing unit. Only seconds may be fractional."""
+
+    model_config = _CONFIG
+
+    measure: Identifier
+    unit: PricingUnit
+    value: Quantity
+
+    @model_validator(mode="after")
+    def validate_discrete_quantity(self) -> Self:
+        if self.unit != "second" and self.value != self.value.to_integral_value():
+            raise ValueError("Discrete priced quantities require integral values")
+        return self
+
+
+class Multiplier(BaseModel):
+    """A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5."""
+
+    model_config = _CONFIG
+
+    name: Identifier
+    value: Identifier
+    factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)]
+
+
+class NamedCost(BaseModel):
+    """A free-form extra charge by name, e.g. azure_model_router_flat_cost."""
+
+    model_config = _CONFIG
+
+    name: Identifier
+    amount: Quantity
+
+
+class PricingSnapshot(BaseModel):
+    """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
+
+    Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
+    resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
+    Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
+    Component sums and float tolerances belong to reconciliation, not this structural contract.
+    """
+
+    model_config = _CONFIG
+
+    schema_version: Literal[1] = 1
+    calculator_version: Identifier
+    model: Identifier
+    provider: Identifier | None
+    pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"]
+    priced_model_key: Identifier
+    multipliers: tuple[Multiplier, ...] = ()
+    rates: tuple[Rate, ...] = ()
+    usage: tuple[PricedQuantity, ...] = ()
+    input_cost: Quantity | None = None
+    output_cost: Quantity | None = None
+    cache_read_cost: Quantity | None = None
+    cache_creation_cost: Quantity | None = None
+    reasoning_cost: Quantity | None = None
+    tool_usage_cost: Quantity | None = None
+    guardrail_cost: Quantity | None = None
+    additional_costs: tuple[NamedCost, ...] = ()
+    original_cost: Quantity | None = None
+    discount_percent: Quantity | None = None
+    discount_amount: Quantity | None = None
+    margin_percent: Quantity | None = None
+    margin_fixed_amount: Quantity | None = None
+    margin_total_amount: Quantity | None = None
+    upstream_reported_cost: Quantity | None = None
+    total_cost: Quantity
+
+    @model_validator(mode="after")
+    def validate_snapshot(self) -> Self:
+        rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates)
+        quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage)
+        if len(rate_keys) != len(self.rates):
+            raise ValueError("Duplicate rate measures and units")
+        if len(quantity_keys) != len(self.usage):
+            raise ValueError("Duplicate priced quantity measures and units")
+        if not rate_keys.issubset(quantity_keys):
+            raise ValueError("Every rate requires one priced quantity with the same measure and unit")
+        if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs):
+            raise ValueError("Duplicate additional cost names")
+        if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers):
+            raise ValueError("Duplicate multiplier names")
+        _check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"})
+        return self
+
+
+class CostEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    usage_event_id: Identifier
+    occurred_at: Instant
+    dimensions: Dimensions
+    charges: Metrics
+    snapshot: PricingSnapshot
+
+    @model_validator(mode="after")
+    def validate_charge(self) -> Self:
+        if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND:
+            raise ValueError("A cost event requires one authoritative USD spend metric")
+        if self.event_id == self.usage_event_id:
+            raise ValueError("Cost and usage events require distinct identities")
+        if self.snapshot.total_cost != self.charges[0].value:
+            raise ValueError("Snapshot total must equal the authoritative spend")
+        return self
+
+
+class Unpriced(BaseModel):
+    model_config = _CONFIG
+
+    usage_event_id: Identifier
+    reason: Identifier
+
+
+class Measurement(BaseModel):
+    model_config = _CONFIG
+
+    source_event_id: Identifier
+    occurred_at: Instant
+    dimensions: Dimensions
+    metric: Metric
+
+
+class DimensionFilter(BaseModel):
+    model_config = _CONFIG
+
+    name: DimensionName
+    values: Annotated[tuple[Identifier, ...], Field(min_length=1)]
+
+
+class AggregateQuery(BaseModel):
+    model_config = _CONFIG
+
+    metric: MetricName
+    filters: tuple[DimensionFilter, ...]
+    group_by: tuple[DimensionName, ...]
+    start: Instant
+    end: Instant
+    granularity: Granularity | None
+
+    @model_validator(mode="after")
+    def validate_query(self) -> Self:
+        _interval(self.start, self.end)
+        if len(frozenset(item.name for item in self.filters)) != len(self.filters):
+            raise ValueError("Use one IN filter per dimension; filters combine with AND")
+        if len(frozenset(self.group_by)) != len(self.group_by):
+            raise ValueError("Duplicate group-by dimensions")
+        if any(
+            (instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0)
+            for instant in (self.start, self.end)
+        ):
+            raise ValueError("Initial reporting queries require UTC day boundaries")
+        return self
+
+
+class AggregateRow(BaseModel):
+    model_config = _CONFIG
+
+    start: Instant
+    end: Instant
+    dimensions: Dimensions
+    value: Quantity
+
+    @model_validator(mode="after")
+    def validate_bounds(self) -> Self:
+        _interval(self.start, self.end)
+        return self
+
+
+def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]:
+    """Validate [start, end) buckets per group; overlapping tag groups are not additive."""
+    groups: Final = frozenset(frozenset(row.dimensions) for row in rows)
+    for group in groups:
+        ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start)
+        if any(left.end > right.start for left, right in zip(ordered, ordered[1:])):
+            raise ValueError("Aggregate buckets overlap within a dimension group")
+    return rows
+
+
+class AdmissionDecision(BaseModel):
+    model_config = _CONFIG
+
+    allowed: bool
+    reason: Identifier | None = None

litellm/caching/in_memory_cache.py

@@ -274,7 +274,9 @@ class InMemoryCache(BaseCache):
     ) -> list[float] | None:
         results: Final = []
         for increment in increment_list:
-            result = await self.async_increment(increment["key"], increment["increment_value"], **kwargs)
+            result = await self.async_increment(
+                increment["key"], increment["increment_value"], ttl=increment["ttl"], **kwargs
+            )
             results.append(result)
         return results
 

litellm/constants.py

@@ -129,6 +129,7 @@ MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
 S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
 S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
 MAX_FILE_LIST_LIMIT: Final = 10000
+ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
 GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10

litellm/images/main.py

@@ -1,9 +1,9 @@
 import asyncio
 import contextvars
 import importlib
-from collections.abc import Coroutine, Mapping
+from collections.abc import Collection, Coroutine, Mapping
 from functools import partial
-from typing import TYPE_CHECKING, Final, Literal, cast, overload
+from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
 
 from pydantic import TypeAdapter, ValidationError
 
@@ -61,6 +61,7 @@ from litellm.types.utils import (
     ProviderRoutingHints,
 )
 from litellm.utils import (
+    EXTRA_BODY_ROUTING_KEYS,
     ImageResponse,
     ProviderConfigManager,
     get_llm_provider,
@@ -466,7 +467,7 @@ def image_generation(
             litellm.LlmProviders.QWENCLOUD,
             litellm.LlmProviders.QWEN_AI_PLATFORM,
             litellm.LlmProviders.EDENAI,
-        ):
+        ) or (custom_llm_provider == litellm.LlmProviders.TENCENT and image_generation_config is not None):
             if image_generation_config is None:
                 raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
 
@@ -807,6 +808,32 @@ def image_variation(
     return response
 
 
+_IMAGE_EDIT_OPENAI_PARAMS: Final = frozenset(get_type_hints(ImageEditOptionalRequestParams))
+_DROP_PARAM_KWARGS: Final = frozenset({"drop_params", "additional_drop_params"})
+_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter[tuple[str, ...]](tuple[str, ...])
+
+
+def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]:
+    try:
+        return _ADDITIONAL_DROP_PARAMS.validate_python(additional_drop_params)
+    except ValidationError:
+        return ()
+
+
+def _tokenhub_passthrough_edit_params(
+    non_default_params: Mapping[str, object],
+    extra_body: Mapping[str, object] | None,
+    mapped_params: Collection[str],
+    additional_drop_params: Collection[str] | None,
+) -> Mapping[str, object]:
+    dropped: Final = frozenset(additional_drop_params or ())
+    unmapped_excluded: Final = _DROP_PARAM_KWARGS | _IMAGE_EDIT_OPENAI_PARAMS | frozenset(mapped_params) | dropped
+    return {
+        **{key: value for key, value in non_default_params.items() if key not in unmapped_excluded},
+        **{key: value for key, value in (extra_body or {}).items() if key not in dropped | EXTRA_BODY_ROUTING_KEYS},
+    }
+
+
 @client
 def image_edit(
     image: FileTypes | list[FileTypes] | None = None,
@@ -935,22 +962,24 @@ def image_edit(
             raise ValueError(f"image edit is not supported for {custom_llm_provider}")
 
         local_vars.update(kwargs)
+        provider_supported_params: Final[frozenset[str]] = frozenset(
+            image_edit_provider_config.get_supported_openai_params(model)
+        )
         # Get ImageEditOptionalRequestParams with only valid parameters
         image_edit_optional_params: Final[ImageEditOptionalRequestParams] = (
             _get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
                 local_vars,
-                provider_supported_params=frozenset(
-                    image_edit_provider_config.get_supported_openai_params(model)
-                ).intersection(non_default_params),
+                provider_supported_params=provider_supported_params.intersection(non_default_params),
             )
         )
         # Get optional parameters for the responses API
+        additional_drop_params: Final = _listed_drop_params(kwargs.get("additional_drop_params"))
         image_edit_request_params: Final[dict] = _get_ImageEditRequestUtils().get_optional_params_image_edit(
             model=model,
             image_edit_provider_config=image_edit_provider_config,
             image_edit_optional_params=image_edit_optional_params,
             drop_params=kwargs.get("drop_params"),
-            additional_drop_params=kwargs.get("additional_drop_params"),
+            additional_drop_params=additional_drop_params,
         )
 
         if image_edit_provider_config.use_multipart_form_data() and (
@@ -1039,7 +1068,19 @@ def image_edit(
             image=images,
             prompt=prompt,
             image_edit_provider_config=image_edit_provider_config,
-            image_edit_optional_request_params=image_edit_request_params,
+            image_edit_optional_request_params={
+                **image_edit_request_params,
+                **(
+                    _tokenhub_passthrough_edit_params(
+                        non_default_params,
+                        extra_body if isinstance(extra_body, dict) else None,
+                        provider_supported_params,
+                        additional_drop_params,
+                    )
+                    if custom_llm_provider == litellm.LlmProviders.TENCENT.value
+                    else {}
+                ),
+            },
             custom_llm_provider=custom_llm_provider,
             litellm_params=litellm_params,
             logging_obj=litellm_logging_obj,

litellm/images/utils.py

@@ -1,6 +1,9 @@
+import base64
+import os
 from collections.abc import Collection, Mapping
 from io import BufferedReader, BytesIO
-from typing import Any, Final, cast, get_type_hints
+from pathlib import Path
+from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
 
 import litellm
 from litellm.litellm_core_utils.token_counter import get_image_type
@@ -16,7 +19,7 @@ class ImageEditRequestUtils:
         image_edit_provider_config: BaseImageEditConfig,
         image_edit_optional_params: ImageEditOptionalRequestParams,
         drop_params: bool | None = None,
-        additional_drop_params: list[str] | None = None,
+        additional_drop_params: Collection[str] | None = None,
     ) -> dict:
         """
         Get optional parameters for the image edit API.
@@ -145,3 +148,36 @@ class ImageEditRequestUtils:
         except Exception:
             # If anything goes wrong, default to PNG
             return FILE_MIME_TYPES[FileType.PNG]
+
+
+@runtime_checkable
+class _SeekableBinaryReader(Protocol):
+    def tell(self) -> int: ...
+
+    def seek(self, offset: int) -> int: ...
+
+    def read(self) -> bytes: ...
+
+
+def _read_image_bytes(image: object) -> bytes:
+    if isinstance(image, bytes):
+        return image
+    if isinstance(image, tuple):
+        return _read_image_bytes(image[1])
+    if isinstance(image, os.PathLike):
+        return Path(image).read_bytes()
+    if isinstance(image, _SeekableBinaryReader):
+        position: Final = image.tell()
+        image.seek(0)
+        data: Final = image.read()
+        image.seek(position)
+        return data
+    raise ValueError(f"Unsupported image type for image edit: {type(image).__name__}")
+
+
+def image_to_data_url(image: object) -> str:
+    if isinstance(image, str):
+        return image
+    image_bytes: Final = _read_image_bytes(image)
+    mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
+    return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"

litellm/integrations/otel/model/spans.py

@@ -338,6 +338,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_ToolTable",
         "LiteLLM_TelemetryReport",
         "LiteLLM_AccessGroupTable",
+        "LiteLLM_AccountingDelivery",
+        "LiteLLM_AccountingEvent",
         "LiteLLM_ClaudeCodePluginTable",
         "LiteLLM_MemoryTable",
         "LiteLLM_AdaptiveRouterState",
@@ -434,6 +436,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
         "reset_budget_windows": PostgresOperation("select", None),
         "write_budget_windows": PostgresOperation("update", None),
         "roll_window_spend_row": PostgresOperation("update", None),
+        "open_window_spend_row": PostgresOperation("upsert", None),
         "seed_window_spend": PostgresOperation("select", None),
         "select_window_spend_rows": PostgresOperation("select", None),
         "commit_window_spend_updates": PostgresOperation("upsert", None),

litellm/integrations/websearch_interception/handler.py

@@ -97,6 +97,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
 
 _RESPONSE_CONTENT_FIELD: Final = "content"
 
+_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
+
 _ResponseT: Final = TypeVar("_ResponseT")
 
 
@@ -363,7 +365,8 @@ class WebSearchInterceptionLogger(CustomLogger):
             get_last_user_message,
         )
 
-        query: Final = get_last_user_message(cast(list[AllMessageValues], messages))
+        last_user_message: Final = get_last_user_message(cast(list[AllMessageValues], messages)) or ""
+        query: Final = last_user_message.removeprefix(_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX).strip()
         if not query:
             return None
 

litellm/litellm_core_utils/core_helpers.py

@@ -266,6 +266,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "tool_use": "tool_calls",
     "refusal": "content_filter",
     "compaction": "length",
+    "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
     "ERROR_TOXIC": "content_filter",

litellm/litellm_core_utils/get_litellm_params.py

@@ -75,6 +75,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "rpm",
             "itpm",
             "otpm",
+            "tpd",
+            "rpd",
             "use_xai_oauth",
             "chatgpt_auth_file",
             "github_copilot_auth_type",

litellm/litellm_core_utils/litellm_logging.py

@@ -2758,7 +2758,7 @@ class Logging(LiteLLMLoggingBaseClass):
         if entry is None or not is_str_keyed_dict(litellm_params):
             return
         metadata: Final[object] = litellm_params.get("metadata")
-        bucket: Final = metadata if is_str_keyed_dict(metadata) else {}
+        bucket: Final = {**metadata} if is_str_keyed_dict(metadata) else {}
         litellm_params["metadata"] = bucket
         with_guardrail_entry(bucket, entry)
 
@@ -3481,7 +3481,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             start_time=start_time,
                             end_time=end_time,
                             print_verbose=print_verbose,
-                            level=LogfireLevel.INFO.value,
+                            level=LogfireLevel.INFO,
                         )
 
                     if callback == "lunary" and lunaryLogger is not None:
@@ -4442,7 +4442,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             response_obj=result,
                             start_time=start_time,
                             end_time=end_time,
-                            level=LogfireLevel.ERROR.value,
+                            level=LogfireLevel.ERROR,
                             print_verbose=print_verbose,
                         )
 

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from datetime import datetime, timezone, tzinfo
 from types import MappingProxyType
-from typing import Final, Literal, TypedDict, cast
+from typing import Final, Literal, Protocol, TypedDict, cast
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from typing_extensions import ReadOnly
@@ -2085,6 +2085,41 @@ def calculate_image_response_web_search_cost(
     )
 
 
+class _ModelPricedImageCostCalculator(Protocol):
+    def __call__(self, model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: ...
+
+
+def _model_priced_image_cost_calculator(
+    custom_llm_provider: str | None, model: str
+) -> _ModelPricedImageCostCalculator | None:
+    match custom_llm_provider:
+        case litellm.LlmProviders.RECRAFT.value:
+            from litellm.llms.recraft.cost_calculator import cost_calculator as recraft_image_cost_calculator
+
+            return recraft_image_cost_calculator
+        case litellm.LlmProviders.AIML.value:
+            from litellm.llms.aiml.image_generation.cost_calculator import cost_calculator as aiml_image_cost_calculator
+
+            return aiml_image_cost_calculator
+        case litellm.LlmProviders.COMETAPI.value:
+            from litellm.llms.cometapi.image_generation.cost_calculator import (
+                cost_calculator as cometapi_image_cost_calculator,
+            )
+
+            return cometapi_image_cost_calculator
+        case litellm.LlmProviders.RUNWAYML.value:
+            from litellm.llms.runwayml.cost_calculator import cost_calculator as runwayml_image_cost_calculator
+
+            return runwayml_image_cost_calculator
+        case litellm.LlmProviders.TENCENT.value:
+            from litellm.llms.tencent.cost_calculator import image_cost_calculator as tencent_image_cost_calculator
+            from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
+
+            return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
+        case _:
+            return None
+
+
 class CostCalculatorUtils:
     @staticmethod
     def call_type_has_image_response(call_type: str) -> bool:
@@ -2131,9 +2166,6 @@ class CostCalculatorUtils:
         from litellm.llms.gemini.image_generation.cost_calculator import (
             cost_calculator as gemini_image_cost_calculator,
         )
-        from litellm.llms.recraft.cost_calculator import (
-            cost_calculator as recraft_image_cost_calculator,
-        )
         from litellm.llms.vertex_ai.image_generation.cost_calculator import (
             cost_calculator as vertex_ai_image_cost_calculator,
         )
@@ -2146,7 +2178,10 @@ class CostCalculatorUtils:
         )
         resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
         pricing: Final = deployment_pricing(model_info)
+        model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
 
+        if model_priced_calculator is not None:
+            return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
         if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
             if isinstance(completion_response, ImageResponse):
                 return vertex_ai_image_cost_calculator(
@@ -2164,36 +2199,6 @@ class CostCalculatorUtils:
                     optional_params=optional_params,
                 )
             raise TypeError("completion_response must be of type ImageResponse for bedrock image cost calculation")
-        elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
-            from litellm.llms.recraft.cost_calculator import (
-                cost_calculator as recraft_image_cost_calculator,
-            )
-
-            return recraft_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
-        elif custom_llm_provider == litellm.LlmProviders.AIML.value:
-            from litellm.llms.aiml.image_generation.cost_calculator import (
-                cost_calculator as aiml_image_cost_calculator,
-            )
-
-            return aiml_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
-        elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
-            from litellm.llms.cometapi.image_generation.cost_calculator import (
-                cost_calculator as cometapi_image_cost_calculator,
-            )
-
-            return cometapi_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
         elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
             if call_type in (
                 CallTypes.image_edit.value,
@@ -2237,16 +2242,6 @@ class CostCalculatorUtils:
                 optional_params=optional_params,
                 model_info=pricing,
             )
-        elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
-            from litellm.llms.runwayml.cost_calculator import (
-                cost_calculator as runwayml_image_cost_calculator,
-            )
-
-            return runwayml_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
         elif (
             custom_llm_provider == litellm.LlmProviders.OPENAI.value
             or custom_llm_provider == litellm.LlmProviders.AZURE.value

litellm/litellm_core_utils/safe_json_dumps.py

@@ -34,7 +34,7 @@ def safe_json_structure(
     def _transform(key: str | None, value: str) -> str:
         return value if value_transform is None else value_transform(key, value)
 
-    def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
+    def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
         # Check for maximum depth.
         if depth > max_depth:
             return "MaxDepthExceeded"
@@ -48,32 +48,32 @@ def safe_json_structure(
         if id(obj) in seen:
             return "CircularReference Detected"
         seen.add(id(obj))
-        result: dict | list | tuple | set | str
         if isinstance(obj, dict):
-            result = {}
-            for k, v in obj.items():
-                if isinstance(k, (str)):
-                    clean_k = k.replace("\x00", "") if "\x00" in k else k
-                    result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
+            mapping_result: Final = {
+                (clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
+                for k, v in obj.items()
+                if isinstance(k, str)
+            }
             seen.remove(id(obj))
-            return result
+            return mapping_result
         elif isinstance(obj, list):
-            result = [_serialize(item, seen, depth + 1, key) for item in obj]
+            list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
             seen.remove(id(obj))
-            return result
+            return list_result
         elif isinstance(obj, tuple):
-            result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
+            tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
             seen.remove(id(obj))
-            return result
+            return tuple_result
         elif isinstance(obj, set):
-            result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
+            sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
+            set_result: Final = sorted(sortable_values)
             seen.remove(id(obj))
-            return result
+            return set_result
         elif isinstance(obj, BaseModel):
             dumped: Final = obj.model_dump()
-            result = _serialize(dumped, seen, depth + 1, key)
+            model_result: Final = _serialize(dumped, seen, depth + 1, key)
             seen.remove(id(obj))
-            return result
+            return model_result
         else:
             # Fall back to string conversion for non-serializable objects.
             try:

litellm/llms/anthropic/messages/caller_model_checks.py

@@ -268,6 +268,7 @@ async def caller_within_model_budget(
                 user_id=user_id,
                 user_model_max_budget=user_model_max_budget,
                 model=model,
+                team_id=user_api_key_auth.team_id,
             )
         except litellm.BudgetExceededError:
             return "denied"
@@ -309,6 +310,7 @@ async def caller_within_model_budget(
                 end_user_id=end_user_id,
                 end_user_model_max_budget=end_user_model_max_budget,
                 model=model,
+                team_id=user_api_key_auth.team_id,
             )
         except litellm.BudgetExceededError:
             return "denied"

litellm/llms/base_llm/files/transformation.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Union
+from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Final, Union
 
 import httpx
 from openai.types.file_deleted import FileDeleted
@@ -38,6 +39,22 @@ else:
     UserAPIKeyAuth = Any
 
 
+def _routes_nowhere(_model_name: str) -> bool:
+    return False
+
+
+@dataclass(frozen=True, slots=True)
+class FileListModelScope:
+    model_mapping_keys: frozenset[str]
+    routes_stored_model_name: Callable[[str], bool] = _routes_nowhere
+
+    def holds(self, model_mapping_keys: Iterable[str], stored: bool) -> bool:
+        mapped_keys: Final = frozenset(model_mapping_keys)
+        return not self.model_mapping_keys.isdisjoint(mapped_keys) or (
+            stored and any(map(self.routes_stored_model_name, mapped_keys))
+        )
+
+
 class BaseFileUploadStream(ABC):
     """Re-iterable request body that yields an upload's bytes lazily.
 
@@ -274,6 +291,8 @@ class BaseFileEndpoints(ABC):
         user_api_key_dict: UserAPIKeyAuth,
         limit: int | None = None,
         after: str | None = None,
+        model_scope: FileListModelScope | None = None,
+        before: str | None = None,
         **data: dict,
     ) -> FileListPage:
         pass

litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py

@@ -10,6 +10,8 @@ import time
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -34,6 +36,15 @@ else:
     LiteLLMLoggingObj = Any
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _PegasusResponse(TypedDict):
+    message: ReadOnly[NotRequired[str | None]]
+    finishReason: ReadOnly[NotRequired[str | None]]
+
+
+_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
+
+
 class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
     """
     Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
@@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
         ModelResponse with choices[0].message.content and finish_reason
         """
         try:
-            completion_response: Final = raw_response.json()
+            completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
         except Exception as e:
             raise BedrockError(
                 message=f"Error parsing response: {raw_response.text}, error: {e}",
@@ -224,7 +235,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
         message_content: Final = completion_response.get("message", "")
 
         # Extract finish reason and map to LiteLLM format
-        finish_reason_raw: Final = completion_response.get("finishReason", "stop")
+        finish_reason_raw: Final = completion_response.get("finishReason") or "stop"
         finish_reason: Final = map_finish_reason(finish_reason_raw)
 
         # Set the response content

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -1,11 +1,12 @@
 import copy
 import json
 import time
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, cast, get_args
 
 import httpx
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 
 _GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
 
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CohereGeneration(TypedDict):
+    text: ReadOnly[str | None]
+    finish_reason: ReadOnly[str | None]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CohereInvokeResponse(TypedDict):
+    text: NotRequired[ReadOnly[str | None]]
+    generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _LlamaInvokeResponse(TypedDict):
+    generation: ReadOnly[str | None]
+
+
+_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
+_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
+
 _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
     "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
 )
@@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         outputText: str | None = None
         try:
             if provider == "cohere":
-                if "text" in completion_response:
-                    outputText = completion_response["text"]
-                elif "generations" in completion_response:
-                    outputText = completion_response["generations"][0]["text"]
+                cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
+                if "text" in cohere_response:
+                    outputText = cohere_response["text"]
+                elif "generations" in cohere_response:
+                    outputText = cohere_response["generations"][0]["text"]
                     model_response.choices[0].finish_reason = map_finish_reason(
-                        completion_response["generations"][0]["finish_reason"]
+                        cohere_response["generations"][0]["finish_reason"] or "stop"
                     )
             elif provider == "anthropic":
                 return litellm.AmazonAnthropicClaudeConfig().transform_response(
@@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
             elif provider == "ai21":
                 outputText = completion_response.get("completions")[0].get("data").get("text")
             elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
-                outputText = completion_response["generation"]
+                outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
             elif provider == "mistral":
                 outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
             else:  # amazon titan

litellm/llms/bedrock/realtime/transformation.py

@@ -53,6 +53,7 @@ from litellm.types.realtime import (
     RealtimeResponseTypedDict,
 )
 
+_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
 _JSON_VALUE: Final = TypeAdapter(object)
 _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
 
@@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             List of Bedrock format messages (JSON strings)
         """
         try:
-            json_message: Final[dict[str, JsonValue]] = json.loads(message)
+            json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
         except json.JSONDecodeError:
             verbose_logger.warning("Invalid JSON message: %s", message[:200])
             return []

litellm/llms/custom_httpx/aiohttp_transport.py

@@ -6,6 +6,7 @@ import ssl
 import sys
 import typing
 import urllib.request
+import weakref
 from collections.abc import Callable, Generator
 from typing import ClassVar, Final
 
@@ -109,6 +110,24 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
         raise mapped_exc(message) from exc
 
 
+def _release_transport_anchor(_transport: object) -> None:
+    """Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
+
+
+def _anchor_transport_to(stream: "AiohttpResponseStream", transport: object) -> None:
+    """Keep the transport alive for as long as the stream can still read.
+
+    The transport finalizer closes the session it owns, and nothing on the path from a
+    live response back to that session references the transport, so a client dropped
+    mid-stream would close the session under the body. ``weakref.finalize`` holds the
+    transport in its own registry rather than on the stream, which keeps the transport
+    out of the response's reference cycle (an exception carrying the response, for one)
+    so it is finalized by refcount once the anchor drops and can still schedule an async
+    close, instead of being collected in the same cyclic pass as the session it closes.
+    """
+    weakref.finalize(stream, _release_transport_anchor, transport)
+
+
 class AiohttpResponseStream(httpx.AsyncByteStream):
     CHUNK_SIZE = 1024 * 16
 
@@ -463,10 +482,12 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
                 # Re-raise if it's a different RuntimeError
                 raise
 
+        stream: Final = AiohttpResponseStream(response)
+        _anchor_transport_to(stream, self)
         return httpx.Response(
             status_code=response.status,
             headers=response.headers,
-            stream=AiohttpResponseStream(response),
+            stream=stream,
             request=request,
         )
 

litellm/llms/dashscope/embed/transformation.py

@@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import ensure_api_route
@@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
 DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _DashScopeUsage(TypedDict, total=False):
+    prompt_tokens: NotRequired[ReadOnly[int]]
+    total_tokens: NotRequired[ReadOnly[int]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _DashScopeEmbeddingResponse(TypedDict, total=False):
+    data: NotRequired[ReadOnly[list[object]]]
+    model: NotRequired[ReadOnly[str | None]]
+    usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
+    id: NotRequired[ReadOnly[object]]
+
+
+_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
+
+
 class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
     """
     Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
@@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
                 message=message,
             )
 
+        parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
         model_response.object = "list"
-        model_response.data = response_json.get("data", [])
-        model_response.model = response_json.get("model", model)
+        model_response.data = parsed_response.get("data", [])
+        model_response.model = parsed_response.get("model", model)
 
-        usage: Final = response_json.get("usage") or {}
+        usage: Final = parsed_response.get("usage") or {}
         prompt_tokens: Final = usage.get("prompt_tokens", 0)
         total_tokens: Final = usage.get("total_tokens", prompt_tokens)
         setattr(
@@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
             ),
         )
 
-        if "id" in response_json:
-            setattr(model_response, "id", response_json["id"])
+        if "id" in parsed_response:
+            setattr(model_response, "id", parsed_response["id"])
 
         return model_response
 

litellm/llms/databricks/streaming_utils.py

@@ -1,6 +1,8 @@
 import json
 from typing import Final
 
+from pydantic import ConfigDict, TypeAdapter
+
 import litellm
 from litellm import verbose_logger
 from litellm.types.llms.openai import (
@@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
     )
 
 
+_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
+
 class ModelResponseIterator:
     def __init__(self, streaming_response, sync_stream: bool):
         self.streaming_response = streaming_response
@@ -99,7 +104,7 @@ class ModelResponseIterator:
             self.response_iterator = self.streaming_response
         chunk: Final = self._next_sse_data()
         try:
-            json_chunk: Final = json.loads(chunk)
+            json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
             return self.chunk_parser(chunk=json_chunk)
         except ValueError as e:
             verbose_logger.debug(
@@ -126,7 +131,7 @@ class ModelResponseIterator:
         if chunk == "[DONE]":
             raise StopAsyncIteration
         try:
-            json_chunk: Final = json.loads(chunk)
+            json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
             return self.chunk_parser(chunk=json_chunk)
         except ValueError as e:
             verbose_logger.debug(

litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py

@@ -4,13 +4,14 @@ from typing import Final
 
 from httpx._types import RequestFiles
 
+from litellm.images.utils import image_to_data_url
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.images.main import ImageEditOptionalRequestParams
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import FileTypes
 
-from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url
+from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig
 
 FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth"
 SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size")
@@ -68,7 +69,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
         )
         request_body: Final[dict[str, object]] = {  # mutable-ok: base class contract returns a dict
             "prompt": prompt,
-            "image_url": to_data_url(next(iter(images))),
+            "image_url": image_to_data_url(next(iter(images))),
             **provider_params,
         }
         return request_body, ()

litellm/llms/fal_ai/image_edit/transformation.py

@@ -1,15 +1,12 @@
-import base64
-import os
 from collections.abc import Mapping
-from pathlib import Path
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
+from typing import TYPE_CHECKING, Final
 
 import httpx
 from httpx._types import RequestFiles
 from pydantic import ConfigDict, TypeAdapter
 
-from litellm.images.utils import ImageEditRequestUtils
+from litellm.images.utils import image_to_data_url
 from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
 from litellm.llms.fal_ai.image_generation.gpt_image_2_transformation import (
     map_gpt_image_quality,
@@ -39,39 +36,6 @@ PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType(
 )
 
 
-@runtime_checkable
-class _SeekableBinaryReader(Protocol):
-    def tell(self) -> int: ...
-
-    def seek(self, offset: int) -> int: ...
-
-    def read(self) -> bytes: ...
-
-
-def _read_image_bytes(image: object) -> bytes:
-    if isinstance(image, bytes):
-        return image
-    if isinstance(image, tuple):
-        return _read_image_bytes(image[1])
-    if isinstance(image, os.PathLike):
-        return Path(image).read_bytes()
-    if isinstance(image, _SeekableBinaryReader):
-        position: Final = image.tell()
-        image.seek(0)
-        data: Final = image.read()
-        image.seek(position)
-        return data
-    raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}")
-
-
-def to_data_url(image: object) -> str:
-    if isinstance(image, str):
-        return image
-    image_bytes: Final = _read_image_bytes(image)
-    mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
-    return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
-
-
 def _first(value: object) -> object:
     return value[0] if isinstance(value, list) and value else value
 
@@ -146,14 +110,14 @@ class FalAIImageEditConfig(BaseImageEditConfig):
             raise ValueError("Fal AI image edit requires at least one input image")
         mask: Final = _first(image_edit_optional_request_params.get("mask"))
         mask_field: Final[Mapping[str, str]] = (
-            MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({})
+            MappingProxyType({"mask_url": image_to_data_url(mask)}) if mask is not None else MappingProxyType({})
         )
         provider_params: Final[Mapping[str, object]] = MappingProxyType(
             {key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
         )
         request_body: Final[dict[str, object]] = {  # mutable-ok: base class contract returns a dict
             "prompt": prompt,
-            "image_urls": tuple(to_data_url(img) for img in images),
+            "image_urls": tuple(image_to_data_url(img) for img in images),
             **mask_field,
             **provider_params,
         }

litellm/llms/gigachat/authenticator.py

@@ -12,6 +12,8 @@ from types import MappingProxyType
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
 from litellm.caching.caching import InMemoryCache
@@ -37,6 +39,17 @@ _token_cache: Final = InMemoryCache()
 _NO_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _GigaChatTokenResponse(TypedDict):
+    tok: NotRequired[ReadOnly[str | None]]
+    access_token: NotRequired[ReadOnly[str | None]]
+    exp: NotRequired[ReadOnly[int | float | str | None]]
+    expires_at: NotRequired[ReadOnly[int | float | str | None]]
+
+
+_TOKEN_RESPONSE: Final = TypeAdapter(_GigaChatTokenResponse)
+
+
 class GigaChatAuthError(BaseLLMException):
     """GigaChat authentication error."""
 
@@ -230,7 +243,7 @@ async def _request_token_async(
 
 def _parse_token_response(response: httpx.Response) -> tuple[str, int]:
     """Parse OAuth token response."""
-    data: Final = response.json()
+    data: Final = _TOKEN_RESPONSE.validate_python(response.json())
 
     # GigaChat returns either 'tok'/'exp' or 'access_token'/'expires_at'
     access_token: Final = data.get("tok") or data.get("access_token")

litellm/llms/openai/realtime/handler.py

@@ -5,7 +5,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy.
 """
 
 import ssl
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
 
 from litellm._logging import redact_string, verbose_logger
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
@@ -23,6 +23,9 @@ from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
 from ..common_utils import is_openai_backed_api_base
 from ..openai import OpenAIChatCompletion
 
+if TYPE_CHECKING:
+    from fastapi import WebSocket
+
 
 class OpenAIRealtime(OpenAIChatCompletion):
     """
@@ -131,7 +134,7 @@ class OpenAIRealtime(OpenAIChatCompletion):
     async def async_realtime(
         self,
         model: str,
-        websocket: Any,
+        websocket: "WebSocket",
         logging_obj: LiteLLMLogging,
         api_base: str | None = None,
         api_key: str | None = None,

litellm/llms/perplexity/embedding/transformation.py

@@ -12,12 +12,14 @@ This module decodes them into float arrays for OpenAI-compatible responses.
 """
 
 import base64
+import builtins
 import struct
-from collections.abc import Mapping
-from typing import Final
+from collections.abc import Mapping, Sequence
+from typing import Final, Literal
 
 import httpx
-from pydantic import ConfigDict, TypeAdapter
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import join_api_base
@@ -31,6 +33,17 @@ _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_i
 _INT: Final = TypeAdapter(int)
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _PerplexityEmbeddingResponse(TypedDict, total=False):
+    model: ReadOnly[str | None]
+    object: ReadOnly[Literal["list"]]
+    data: ReadOnly[Sequence[builtins.object] | Mapping[str, builtins.object] | Literal[""]]
+    usage: ReadOnly[builtins.object]
+
+
+_EMBEDDING_RESPONSE_ADAPTER: Final = TypeAdapter(_PerplexityEmbeddingResponse)
+
+
 class PerplexityEmbeddingError(BaseLLMException):
     def __init__(
         self,
@@ -153,14 +166,18 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
         litellm_params: dict = {},
     ) -> EmbeddingResponse:
         try:
-            raw_response_json: Final = raw_response.json()
+            raw_response_json: Final[object] = raw_response.json()
         except Exception:
             raise PerplexityEmbeddingError(message=raw_response.text, status_code=raw_response.status_code)
 
-        model_response.model = raw_response_json.get("model", model)
-        model_response.object = raw_response_json.get("object", "list")
+        response_data: Final[_PerplexityEmbeddingResponse] = _EMBEDDING_RESPONSE_ADAPTER.validate_python(
+            raw_response_json
+        )
 
-        raw_data: Final = raw_response_json.get("data", [])
+        model_response.model = response_data.get("model", model)
+        model_response.object = response_data.get("object", "list")
+
+        raw_data: Final = response_data.get("data", [])
         decoded_data: Final[list[dict[str, object]]] = []
         for item in raw_data:
             decoded_item = dict(_JSON_OBJECT.validate_python(item))
@@ -168,7 +185,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig):
             decoded_data.append(decoded_item)
         model_response.data = decoded_data
 
-        usage_data: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
+        usage_data: Final = _JSON_OBJECT.validate_python(response_data.get("usage", {}))
         usage: Final = Usage(
             prompt_tokens=_INT.validate_python(
                 usage_data.get("prompt_tokens", 0) or usage_data.get("total_tokens", 0) or 0

litellm/llms/tencent/cost_calculator.py

@@ -1,6 +1,49 @@
-from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
-from litellm.types.utils import Usage
+from typing import Final
+
+from litellm.litellm_core_utils.llm_cost_calc.utils import (
+    calculate_image_response_cost_from_usage,
+    flat_image_cost,
+    generic_cost_per_token,
+    resolve_image_model_info,
+)
+from litellm.types.utils import ImageResponse, ModelInfo, Usage
 
 
 def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
     return generic_cost_per_token(model=model, usage=usage, custom_llm_provider="tencent")
+
+
+def _deployment_image_token_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
+    if model_info is None or model_info.get("output_cost_per_image_token") is not None:
+        return model_info
+    output_cost_per_token: Final = model_info.get("output_cost_per_token")
+    if output_cost_per_token is None:
+        return model_info
+    priced: Final[ModelInfo] = {**model_info, "output_cost_per_image_token": output_cost_per_token}
+    return priced
+
+
+def _deployment_prices_per_image_only(model_info: ModelInfo | None) -> bool:
+    return (
+        model_info is not None
+        and model_info.get("output_cost_per_image") is not None
+        and model_info.get("output_cost_per_image_token") is None
+        and model_info.get("output_cost_per_token") is None
+    )
+
+
+def image_cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
+    resolved_model_info: Final = resolve_image_model_info(
+        model=model, custom_llm_provider="tencent", model_info=_deployment_image_token_pricing(model_info)
+    )
+    if _deployment_prices_per_image_only(model_info):
+        return flat_image_cost(resolved_model_info, image_response)
+    token_based_cost: Final = calculate_image_response_cost_from_usage(
+        model=model,
+        image_response=image_response,
+        custom_llm_provider="tencent",
+        model_info=resolved_model_info,
+    )
+    if token_based_cost is not None:
+        return token_based_cost
+    return flat_image_cost(resolved_model_info, image_response)

litellm/llms/tencent/image_edit/transformation.py

@@ -0,0 +1,147 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+import httpx
+from httpx._types import RequestFiles
+from pydantic import TypeAdapter
+from typing_extensions import override
+
+from litellm.images.utils import image_to_data_url
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
+from litellm.llms.tencent.image_generation.transformation import (
+    TOKENHUB_IMAGE_PARAMS,
+    tokenhub_image_headers,
+    tokenhub_image_params,
+    tokenhub_image_response,
+    tokenhub_image_url,
+)
+from litellm.types.images.main import ImageEditOptionalRequestParams
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import ImageResponse
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_IMAGES: Final = TypeAdapter(tuple[object, ...])
+_FORM_BOOLEANS: Final = MappingProxyType({"true": True, "false": False})
+_OPENAI_PARAMS: Final = frozenset(TOKENHUB_IMAGE_PARAMS)
+
+_FormEntries: TypeAlias = tuple[tuple[tuple[str, ...], object], ...]
+
+
+def _form_scalar(value: object) -> object:
+    match value:
+        case str() if value.lower() in _FORM_BOOLEANS:
+            return _FORM_BOOLEANS[value.lower()]
+        case str() if value.removeprefix("-").isdigit():
+            return int(value)
+        case _:
+            return value
+
+
+def _form_path(key: str) -> tuple[str, ...]:
+    root, bracket, rest = key.partition("[")
+    if not bracket or not rest.endswith("]"):
+        return (key,)
+    return (root, *rest.removesuffix("]").split("]["))
+
+
+def _nested_form_value(entries: _FormEntries) -> object:
+    match entries:
+        case (((), value),):
+            return _form_scalar(value)
+        case _:
+            return _nested_form_fields(tuple(entry for entry in entries if entry[0]))
+
+
+def _nested_form_fields(entries: _FormEntries) -> Mapping[str, object]:
+    roots: Final = tuple(dict.fromkeys(path[0] for path, _ in entries))
+    return {
+        root: _nested_form_value(tuple((path[1:], value) for path, value in entries if path[0] == root))
+        for root in roots
+    }
+
+
+def _tokenhub_form_params(params: Mapping[str, object]) -> Mapping[str, object]:
+    tokenhub_entries: Final = tuple(
+        (_form_path(key), value) for key, value in params.items() if key not in _OPENAI_PARAMS
+    )
+    return {
+        **{key: value for key, value in params.items() if key in _OPENAI_PARAMS},
+        **_nested_form_fields(tokenhub_entries),
+    }
+
+
+class TencentImageEditConfig(BaseImageEditConfig):
+    @override
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: mirrors override contract
+        return list(TOKENHUB_IMAGE_PARAMS)
+
+    @override
+    def map_openai_params(
+        self,
+        image_edit_optional_params: ImageEditOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: mirrors override contract
+        return dict(tokenhub_image_params(image_edit_optional_params, model, drop_params))
+
+    @override
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        api_key: str | None = None,
+        litellm_params: Mapping[str, object] | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: mirrors override contract
+        return dict(tokenhub_image_headers(headers, api_key, model))
+
+    @override
+    def get_complete_url(
+        self,
+        model: str,
+        api_base: str | None,
+        litellm_params: Mapping[str, object],
+    ) -> str:
+        return tokenhub_image_url(api_base)
+
+    @override
+    def use_multipart_form_data(self) -> bool:
+        return False
+
+    @override
+    def transform_image_edit_request(
+        self,
+        model: str,
+        prompt: str | None,
+        image: object,
+        image_edit_optional_request_params: Mapping[str, object],
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> tuple[dict[str, object], RequestFiles]:  # mutable-ok: mirrors override contract
+        images: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
+        return {
+            "model": model,
+            "prompt": prompt,
+            "images": tuple(image_to_data_url(img) for img in images if img is not None),
+            **_tokenhub_form_params(image_edit_optional_request_params),
+        }, ()
+
+    @override
+    def transform_image_edit_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "LiteLLMLoggingObj",
+    ) -> ImageResponse:
+        image_response: Final = tokenhub_image_response(raw_response)
+        if image_response is None:
+            raise BaseLLMException(
+                status_code=raw_response.status_code,
+                message=raw_response.text,
+                headers=raw_response.headers,
+            )
+        return image_response

litellm/llms/tencent/image_generation/transformation.py

@@ -0,0 +1,200 @@
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Final
+
+import httpx
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
+from typing_extensions import override
+
+from litellm.exceptions import AuthenticationError, UnsupportedParamsError
+from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.image_generation.transformation import (
+    BaseImageGenerationConfig,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIImageGenerationOptionalParams,
+)
+from litellm.types.utils import (
+    ImageObject,
+    ImageResponse,
+    ImageUsage,
+    ImageUsageInputTokensDetails,
+    LlmProviders,
+)
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+TOKENHUB_API_BASE: Final = "https://tokenhub-intl.tencentcloudmaas.com/v1"
+TOKENHUB_IMAGE_PATH: Final = "/wand/si-image/generation"
+TOKENHUB_IMAGE_MODEL_PREFIX: Final = "seedream-"
+TOKENHUB_IMAGE_PARAMS: Final[tuple[OpenAIImageGenerationOptionalParams, ...]] = (
+    "n",
+    "size",
+    "response_format",
+    "output_format",
+    "background",
+    "user",
+)
+
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+
+
+class _TokenHubImage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    url: str | None = None
+    b64_json: str | None = None
+    size: str | None = None
+    output_format: str | None = None
+
+
+class _TokenHubUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    total_tokens: int
+
+
+class _TokenHubImageResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, hide_input_in_errors=True)
+
+    created: int
+    data: tuple[_TokenHubImage, ...]
+    tokenhub_usage: _TokenHubUsage | None = None
+
+
+def is_tokenhub_image_model(model: str) -> bool:
+    return model.removeprefix(f"{LlmProviders.TENCENT.value}/").startswith(TOKENHUB_IMAGE_MODEL_PREFIX)
+
+
+def tokenhub_image_url(api_base: str | None) -> str:
+    base: Final = api_base or get_secret_str("TENCENT_API_BASE") or TOKENHUB_API_BASE
+    return join_api_base(ensure_api_route(base, "v1"), TOKENHUB_IMAGE_PATH)
+
+
+def tokenhub_image_headers(headers: Mapping[str, str], api_key: str | None, model: str) -> Mapping[str, str]:
+    resolved_api_key: Final = api_key or get_secret_str("TENCENT_API_KEY")
+    if not resolved_api_key:
+        raise AuthenticationError(
+            message="TENCENT_API_KEY is not set. Set it in the environment or pass api_key.",
+            llm_provider="tencent",
+            model=model,
+        )
+    return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
+
+
+def tokenhub_image_params(params: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, object]:
+    n: Final = params.get("n")
+    if n is not None and n != 1 and not drop_params:
+        raise UnsupportedParamsError(
+            status_code=400,
+            message=f"Tencent TokenHub returns one image per request, so n={n} is not supported for {model}. "
+            + "To send n=1 instead, set `litellm.drop_params = True`.",
+        )
+    return {
+        key: value
+        for key, value in params.items()
+        if key != "n" and value is not None and not (key == "background" and value == "auto")
+    }
+
+
+def tokenhub_image_response(raw_response: httpx.Response) -> ImageResponse | None:
+    try:
+        parsed: Final = _TokenHubImageResponse.model_validate_json(raw_response.content)
+    except ValidationError:
+        return None
+    total_tokens: Final = parsed.tokenhub_usage.total_tokens if parsed.tokenhub_usage is not None else 0
+    first_image: Final = parsed.data[0] if parsed.data else _TokenHubImage()
+    return ImageResponse(
+        created=parsed.created,
+        data=[ImageObject(url=image.url, b64_json=image.b64_json) for image in parsed.data],
+        size=first_image.size,
+        output_format=first_image.output_format,
+        usage=ImageUsage(
+            input_tokens=0,
+            input_tokens_details=ImageUsageInputTokensDetails(image_tokens=0, text_tokens=0),
+            output_tokens=total_tokens,
+            total_tokens=total_tokens,
+        ),
+    )
+
+
+class TencentImageGenerationConfig(BaseImageGenerationConfig):
+    @override
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIImageGenerationOptionalParams]:  # mutable-ok: mirrors override contract
+        return list(TOKENHUB_IMAGE_PARAMS)
+
+    @override
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: mirrors override contract
+        return {**optional_params, **tokenhub_image_params(non_default_params, model, drop_params)}
+
+    @override
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        return tokenhub_image_url(api_base)
+
+    @override
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: mirrors override contract
+        return dict(tokenhub_image_headers(headers, api_key, model))
+
+    @override
+    def transform_image_generation_request(
+        self,
+        model: str,
+        prompt: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        headers: Mapping[str, str],
+    ) -> dict[str, object]:  # mutable-ok: mirrors override contract
+        extra_body: Final = _JSON_OBJECT.validate_python(optional_params.get("extra_body") or {})
+        params: Final = {key: value for key, value in optional_params.items() if key != "extra_body"}
+        return {"model": model, "prompt": prompt, **params, **extra_body}
+
+    @override
+    def transform_image_generation_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: ImageResponse,
+        logging_obj: "LiteLLMLoggingObj",
+        request_data: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        encoding: object,
+        api_key: str | None = None,
+        json_mode: bool | None = None,
+    ) -> ImageResponse:
+        image_response: Final = tokenhub_image_response(raw_response)
+        if image_response is None:
+            raise BaseLLMException(
+                status_code=raw_response.status_code,
+                message=raw_response.text,
+                headers=raw_response.headers,
+            )
+        return image_response

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -2743,10 +2743,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 chunk_indices = support.get("groundingChunkIndices", [])
 
                 if start_index is not None and end_index is not None and chunk_indices:
-                    # Use the first chunk's URL for the annotation
-                    first_chunk_idx = chunk_indices[0]
-                    if first_chunk_idx in chunk_to_uri_map:
-                        uri_info = chunk_to_uri_map[first_chunk_idx]
+                    for chunk_index in chunk_indices:
+                        if chunk_index not in chunk_to_uri_map:
+                            continue
+                        uri_info = chunk_to_uri_map[chunk_index]
 
                         url_citation: ChatCompletionAnnotationURLCitation = {
                             "start_index": start_index,

litellm/llms/vertex_ai/vertex_gemma_models/transformation.py

@@ -9,10 +9,11 @@ The actual message transformation reuses OpenAIGPTConfig since Gemma uses OpenAI
 """
 
 from collections.abc import Callable
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
 
 import httpx
-from pydantic import ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError
+from typing_extensions import NotRequired, ReadOnly, TypeAliasType, TypedDict
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
@@ -32,6 +33,17 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
 
+class _PredictionsEnvelope(TypedDict):
+    predictions: NotRequired[ReadOnly[dict[str, object]]]
+
+
+_PredictionsEnvelopeType = TypeAliasType("_PredictionsEnvelopeType", _PredictionsEnvelope)
+_PREDICTIONS_ENVELOPE: Final[TypeAdapter[_PredictionsEnvelope]] = TypeAdapter(
+    _PredictionsEnvelopeType,
+    config=ConfigDict(strict=True, extra="allow"),
+)
+
+
 def parse_vertex_gemma_container_error(predictions: object) -> VertexGemmaContainerError | None:
     try:
         return VertexGemmaContainerError.model_validate(predictions)
@@ -142,7 +154,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
 
     def _unwrap_predictions_response(
         self,
-        response_json: dict[str, Any],
+        response_json: _PredictionsEnvelope,
     ) -> dict[str, object]:
         """
         Unwrap the Vertex Gemma predictions format to OpenAI format.
@@ -354,7 +366,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
                 message=f"Request failed: {response.text}",
             )
 
-        response_json: Final = response.json()
+        response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
 
         # Unwrap predictions to get OpenAI-compatible response
         openai_response: Final = self._unwrap_predictions_response(response_json)
@@ -449,7 +461,7 @@ class VertexGemmaConfig(OpenAIGPTConfig):
                 message=f"Request failed: {response.text}",
             )
 
-        response_json: Final = response.json()
+        response_json: Final = _PREDICTIONS_ENVELOPE.validate_python(response.json())
 
         # Unwrap predictions to get OpenAI-compatible response
         openai_response: Final = self._unwrap_predictions_response(response_json)

litellm/llms/volcengine/responses/transformation.py

@@ -3,6 +3,7 @@ from types import UnionType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, Union, get_args, get_origin
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 from pydantic import fields as pyd_fields
 
 import litellm
@@ -37,6 +38,8 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_RESPONSE_BODY: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
 
 class _EventModelClass(Protocol):
     @property
@@ -420,7 +423,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
     @staticmethod
     def _parsed_response_body(raw_response: httpx.Response) -> dict[str, object]:
-        return raw_response.json()
+        return _RESPONSE_BODY.validate_python(raw_response.json())
 
     @staticmethod
     def _annotation_origin(annotation: object) -> object:

litellm/model_prices_and_context_window_backup.json

@@ -21589,7 +21589,13 @@
         ],
         "supports_vision": true,
         "supports_audio_input": true,
-        "supports_video_input": true
+        "supports_video_input": true,
+        "input_cost_per_token_priority": 5.6252e-07,
+        "output_cost_per_token_priority": 3.37498e-06,
+        "cache_read_input_token_cost_priority": 5.628e-08,
+        "cache_creation_input_token_cost_priority": 5.6252e-07,
+        "input_cost_per_audio_token_priority": 1.12497e-06,
+        "cache_read_input_audio_token_cost_priority": 1.1249e-07
     },
     "databricks/databricks-gemini-3-1-flash-image": {
         "cache_creation_input_token_cost": 6.2503e-07,
@@ -21676,7 +21682,14 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_video_input": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.50002e-06,
+        "output_cost_per_token_priority": 2.699998e-05,
+        "cache_read_input_token_cost_priority": 4.5003e-07,
+        "cache_creation_input_token_cost_priority": 4.50002e-06,
+        "input_cost_per_token_above_200k_tokens_priority": 8.99997e-06,
+        "output_cost_per_token_above_200k_tokens_priority": 4.049997e-05,
+        "cache_read_input_token_cost_above_200k_tokens_priority": 8.9999e-07
     },
     "databricks/databricks-gemini-3-flash": {
         "cache_creation_input_token_cost": 6.2503e-07,
@@ -21710,7 +21723,13 @@
         ],
         "supports_vision": true,
         "supports_audio_input": true,
-        "supports_video_input": true
+        "supports_video_input": true,
+        "input_cost_per_token_priority": 1.12497e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.1249e-07,
+        "cache_creation_input_token_cost_priority": 1.12497e-06,
+        "input_cost_per_audio_token_priority": 2.25001e-06,
+        "cache_read_input_audio_token_cost_priority": 2.2498e-07
     },
     "databricks/databricks-gemini-3-pro": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -21757,7 +21776,11 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-7-flash": {
         "cache_creation_input_token_cost": 7.4998e-07,
@@ -21785,10 +21808,14 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-6-flash": {
-        "cache_creation_input_token_cost": 1.87502e-06,
+        "cache_creation_input_token_cost": 7.4998e-07,
         "cache_read_input_token_cost": 7.497e-08,
         "input_cost_per_token": 7.4998e-07,
         "input_dbu_cost_per_token": 1.0714e-05,
@@ -21816,7 +21843,11 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-5-flash": {
         "cache_creation_input_token_cost": 1.87502e-06,
@@ -21846,7 +21877,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 3.37498e-06,
+        "output_cost_per_token_priority": 2.025002e-05,
+        "cache_read_input_token_cost_priority": 3.3747e-07,
+        "cache_creation_input_token_cost_priority": 3.37498e-06
     },
     "databricks/databricks-gemini-3-5-flash-lite": {
         "cache_creation_input_token_cost": 3.7499e-07,
@@ -21878,7 +21913,11 @@
         "supports_response_schema": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 6.7501e-07,
+        "output_cost_per_token_priority": 5.62499e-06,
+        "cache_read_input_token_cost_priority": 6.748e-08,
+        "cache_creation_input_token_cost_priority": 6.7501e-07
     },
     "databricks/databricks-gemma-3-12b": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -21921,7 +21960,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": false,
+        "input_cost_per_token_priority": 2.45e-06,
+        "output_cost_per_token_priority": 7.7e-06,
+        "cache_read_input_token_cost_priority": 4.55e-07,
+        "cache_creation_input_token_cost_priority": 2.45e-06
     },
     "databricks/databricks-glm-5-3": {
         "cache_creation_input_token_cost": 1.4e-06,
@@ -21951,7 +21994,11 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": false,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 2.45e-06,
+        "output_cost_per_token_priority": 7.7e-06,
+        "cache_read_input_token_cost_priority": 4.55e-07,
+        "cache_creation_input_token_cost_priority": 2.45e-06
     },
     "databricks/databricks-glm-5-3-flash": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -21980,7 +22027,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 2.625e-07,
+        "output_cost_per_token_priority": 8.75e-07,
+        "cache_read_input_token_cost_priority": 5.25e-08,
+        "cache_creation_input_token_cost_priority": 2.625e-07
     },
     "databricks/databricks-gpt-5": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -21999,7 +22050,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 2.49998e-06,
+        "output_cost_per_token_priority": 1.999998e-05,
+        "cache_read_input_token_cost_priority": 2.4997e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06
     },
     "databricks/databricks-gpt-5-1": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -22018,7 +22073,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 2.49998e-06,
+        "output_cost_per_token_priority": 1.999998e-05,
+        "cache_read_input_token_cost_priority": 2.4997e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06
     },
     "databricks/databricks-gpt-5-2": {
         "cache_creation_input_token_cost": 1.75e-06,
@@ -22037,7 +22096,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 3.5e-06,
+        "output_cost_per_token_priority": 2.8e-05,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "cache_creation_input_token_cost_priority": 3.5e-06
     },
     "databricks/databricks-gpt-5-3-codex": {
         "cache_creation_input_token_cost": 1.75e-06,
@@ -22056,7 +22119,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 3.5e-06,
+        "output_cost_per_token_priority": 2.8e-05,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "cache_creation_input_token_cost_priority": 3.5e-06
     },
     "databricks/databricks-gpt-5-4": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -22087,7 +22154,14 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.00003e-06,
+        "output_cost_per_token_priority": 2.999997e-05,
+        "cache_read_input_token_cost_priority": 5.0001e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 5.00003e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.999997e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 5.0001e-07
     },
     "databricks/databricks-gpt-5-4-mini": {
         "cache_creation_input_token_cost": 7.4998e-07,
@@ -22115,7 +22189,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.50003e-06,
+        "output_cost_per_token_priority": 8.99997e-06,
+        "cache_read_input_token_cost_priority": 1.5001e-07,
+        "cache_creation_input_token_cost_priority": 1.50003e-06
     },
     "databricks/databricks-gpt-5-4-nano": {
         "cache_creation_input_token_cost": 1.9999e-07,
@@ -22175,7 +22253,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 8.00002e-06,
+        "output_cost_per_token_priority": 4.000003e-05,
+        "cache_read_input_token_cost_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_priority": 9.99999e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 1.599997e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 6.000001e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.59999e-06,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 1.999998e-05
     },
     "databricks/databricks-gpt-5-6-terra": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -22207,7 +22293,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 2.399999e-05,
+        "cache_read_input_token_cost_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 3.600002e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-5-6-luna": {
         "cache_creation_input_token_cost": 2.4997e-07,
@@ -22239,7 +22333,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 3.9998e-07,
+        "output_cost_per_token_priority": 2.40002e-06,
+        "cache_read_input_token_cost_priority": 3.997e-08,
+        "cache_creation_input_token_cost_priority": 5.0001e-07,
+        "input_cost_per_token_above_272k_tokens_priority": 8.0003e-07,
+        "output_cost_per_token_above_272k_tokens_priority": 3.60003e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.001e-08,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 1.00002e-06
     },
     "databricks/databricks-gpt-5-5": {
         "cache_creation_input_token_cost": 5.00003e-06,
@@ -22270,7 +22372,14 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.249997e-05,
+        "output_cost_per_token_priority": 7.50001e-05,
+        "cache_read_input_token_cost_priority": 1.24999e-06,
+        "cache_creation_input_token_cost_priority": 1.249997e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 1.249997e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 7.50001e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.24999e-06
     },
     "databricks/databricks-gpt-5-5-pro": {
         "cache_creation_input_token_cost": 2.999997e-05,
@@ -22319,7 +22428,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 4.5003e-07,
+        "output_cost_per_token_priority": 3.60003e-06,
+        "cache_read_input_token_cost_priority": 4.501e-08,
+        "cache_creation_input_token_cost_priority": 4.5003e-07
     },
     "databricks/databricks-gpt-5-nano": {
         "cache_creation_input_token_cost": 4.998e-08,
@@ -22469,7 +22582,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.25e-06,
+        "output_cost_per_token_priority": 2.625e-05,
+        "cache_read_input_token_cost_priority": 5.25e-07,
+        "cache_creation_input_token_cost_priority": 5.25e-06
     },
     "databricks/system.ai.kimi-k3": {
         "litellm_provider": "databricks",
@@ -22551,7 +22668,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": false,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 3.85e-07,
+        "output_cost_per_token_priority": 3.85e-06,
+        "cache_read_input_token_cost_priority": 3.85e-07,
+        "cache_creation_input_token_cost_priority": 3.85e-07
     },
     "databricks/databricks-qwen3-next-80b-a3b-instruct": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -61865,6 +61986,39 @@
         "supports_reasoning": true,
         "supports_vision": false
     },
+    "tencent/seedream-image-v5.0-flash": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.018,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://cloud.tencent.com/document/product/1823/130055",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
+    "tencent/seedream-image-v5.0-lite": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.035,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://www.tencentcloud.com/document/product/1300/78937",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
+    "tencent/seedream-image-v5.0-pro": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.09,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://www.tencentcloud.com/document/product/1300/78937",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
     "cognition/swe-1.6": {
         "input_cost_per_token": 5e-07,
         "output_cost_per_token": 2.5e-06,
@@ -77774,7 +77928,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.25e-07,
+        "output_cost_per_token_priority": 2.1e-06,
+        "cache_read_input_token_cost_priority": 5.25e-08,
+        "cache_creation_input_token_cost_priority": 5.25e-07
     },
     "databricks/databricks-deepseek-v4-1-flash-1017": {
         "cache_creation_input_token_cost": 3.0002e-07,
@@ -77897,7 +78055,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_video_input": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.50002e-06,
+        "output_cost_per_token_priority": 2.250003e-05,
+        "cache_read_input_token_cost_priority": 2.2498e-07,
+        "cache_creation_input_token_cost_priority": 4.50002e-06
     },
     "databricks/databricks-gpt-5-4-pro": {
         "cache_creation_input_token_cost": 2.999997e-05,
@@ -77959,7 +78121,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 1.99997e-05,
+        "cache_read_input_token_cost_priority": 1.9999e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-6-1-terra": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -77991,7 +78161,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.99997e-06,
+        "output_cost_per_token_priority": 9.99999e-06,
+        "cache_read_input_token_cost_priority": 1.0003e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 4.00001e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 1.500002e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06
     },
     "databricks/databricks-gpt-6-astra": {
         "cache_creation_input_token_cost": 1.249997e-05,
@@ -78023,7 +78201,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.999998e-05,
+        "output_cost_per_token_priority": 9.99999e-05,
+        "cache_read_input_token_cost_priority": 1.99997e-06,
+        "cache_creation_input_token_cost_priority": 2.500001e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 4.000003e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 0.0001500002,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05
     },
     "databricks/databricks-gpt-6-luna": {
         "cache_creation_input_token_cost": 1.2502e-07,
@@ -78055,7 +78241,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.9999e-07,
+        "output_cost_per_token_priority": 1.0003e-06,
+        "cache_read_input_token_cost_priority": 2.002e-08,
+        "cache_creation_input_token_cost_priority": 2.4997e-07,
+        "input_cost_per_token_above_272k_tokens_priority": 3.9998e-07,
+        "output_cost_per_token_above_272k_tokens_priority": 1.5001e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07
     },
     "databricks/databricks-gpt-6-sol": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -78087,7 +78281,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 1.99997e-05,
+        "cache_read_input_token_cost_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-image-1": {
         "cache_creation_input_token_cost": 5.00003e-06,

litellm/provider_endpoints_support_backup.json

@@ -3793,8 +3793,8 @@
         "responses": true,
         "text_completion": true,
         "embeddings": false,
-        "image_generations": false,
-        "image_edits": false,
+        "image_generations": true,
+        "image_edits": true,
         "audio_transcriptions": false,
         "audio_speech": false,
         "rerank": false,

litellm/proxy/_lazy_openapi_snapshot.json

@@ -3487,6 +3487,13 @@
               "title": "Entities",
               "type": "object"
             },
+            "guardrails": {
+              "additionalProperties": {
+                "$ref": "#/components/schemas/GuardrailMetric"
+              },
+              "title": "Guardrails",
+              "type": "object"
+            },
             "mcp_servers": {
               "additionalProperties": {
                 "$ref": "#/components/schemas/MetricWithMetadata"
@@ -3802,6 +3809,74 @@
           "title": "ExportType",
           "type": "string"
         },
+        "GuardrailKeyMetric": {
+          "properties": {
+            "metadata": {
+              "$ref": "#/components/schemas/KeyMetadata"
+            },
+            "metrics": {
+              "$ref": "#/components/schemas/GuardrailSpendMetrics"
+            }
+          },
+          "required": [
+            "metrics",
+            "metadata"
+          ],
+          "title": "GuardrailKeyMetric",
+          "type": "object"
+        },
+        "GuardrailMetadata": {
+          "properties": {
+            "guardrail_name": {
+              "title": "Guardrail Name",
+              "type": "string"
+            }
+          },
+          "required": [
+            "guardrail_name"
+          ],
+          "title": "GuardrailMetadata",
+          "type": "object"
+        },
+        "GuardrailMetric": {
+          "properties": {
+            "api_key_breakdown": {
+              "additionalProperties": {
+                "$ref": "#/components/schemas/GuardrailKeyMetric"
+              },
+              "title": "Api Key Breakdown",
+              "type": "object"
+            },
+            "metadata": {
+              "$ref": "#/components/schemas/GuardrailMetadata"
+            },
+            "metrics": {
+              "$ref": "#/components/schemas/GuardrailSpendMetrics"
+            }
+          },
+          "required": [
+            "metrics",
+            "metadata"
+          ],
+          "title": "GuardrailMetric",
+          "type": "object"
+        },
+        "GuardrailSpendMetrics": {
+          "properties": {
+            "spend": {
+              "default": 0.0,
+              "title": "Spend",
+              "type": "number"
+            },
+            "untracked_units": {
+              "default": 0,
+              "title": "Untracked Units",
+              "type": "integer"
+            }
+          },
+          "title": "GuardrailSpendMetrics",
+          "type": "object"
+        },
         "HTTPAuthSecurityScheme": {
           "description": "Defines a security scheme using HTTP authentication.",
           "properties": {

litellm/proxy/auth/user_api_key_auth.py

@@ -370,7 +370,7 @@ class _KeyModelBudgetLimiter(Protocol):
 
 class _UserModelBudgetLimiter(Protocol):
     async def is_user_within_model_budget(
-        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str
+        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str, team_id: str | None = None
     ) -> bool: ...
 
 
@@ -388,11 +388,15 @@ class _EveryModelBudgetLimiter(Protocol):
     async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
 
     async def is_user_within_model_budget(
-        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str
+        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str, team_id: str | None = None
     ) -> bool: ...
 
     async def is_end_user_within_model_budget(
-        self, end_user_id: str, end_user_model_max_budget: Mapping[str, object], model: str
+        self,
+        end_user_id: str,
+        end_user_model_max_budget: Mapping[str, object],
+        model: str,
+        team_id: str | None = None,
     ) -> bool: ...
 
     async def is_team_within_model_budget(
@@ -503,6 +507,7 @@ async def _check_user_model_budget(
             user_id=valid_token.user_id,
             user_model_max_budget=user_model_max_budget,
             model=model_name,
+            team_id=valid_token.team_id,
         )
 
 
@@ -547,6 +552,7 @@ async def _is_within_every_model_budget(
                 user_id=valid_token.user_id,
                 user_model_max_budget=user_model_max_budget,
                 model=model,
+                team_id=valid_token.team_id,
             )
         if (
             valid_token.end_user_id is not None
@@ -557,6 +563,7 @@ async def _is_within_every_model_budget(
                 end_user_id=valid_token.end_user_id,
                 end_user_model_max_budget=end_user_model_max_budget,
                 model=model,
+                team_id=valid_token.team_id,
             )
         if valid_token.team_id is not None and isinstance(team_model_max_budget, Mapping) and team_model_max_budget:
             await model_max_budget_limiter.is_team_within_model_budget(
@@ -3154,6 +3161,7 @@ async def validate_resolved_virtual_key(  # noqa: C901  # Preserve ordering of e
                             end_user_id=valid_token.end_user_id,
                             end_user_model_max_budget=end_user_mmb,
                             model=model_name,
+                            team_id=valid_token.team_id,
                         )
 
         # Check 6: Additional Common Checks across jwt + key auth
@@ -4733,6 +4741,7 @@ async def _run_post_custom_auth_checks(
                 end_user_id=valid_token.end_user_id,
                 end_user_model_max_budget=end_user_mmb,
                 model=model_name,
+                team_id=valid_token.team_id,
             )
 
     # team / user / end_user / project context objects are fetched by

litellm/proxy/client/models.py

@@ -3,9 +3,12 @@ from collections.abc import Mapping
 from typing import Any, Final
 
 import requests
+from pydantic import ConfigDict, TypeAdapter
 
 from .exceptions import NotFoundError, UnauthorizedError
 
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
 
 class ModelsManagementClient:
     def __init__(self, base_url: str, api_key: str | None = None, timeout: int = 30):
@@ -109,7 +112,7 @@ class ModelsManagementClient:
         try:
             response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
-            return response.json()
+            return _JSON_OBJECT.validate_python(response.json())
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 401:
                 raise UnauthorizedError(e)
@@ -145,7 +148,7 @@ class ModelsManagementClient:
         try:
             response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
-            return response.json()
+            return _JSON_OBJECT.validate_python(response.json())
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 401:
                 raise UnauthorizedError(e)
@@ -287,7 +290,7 @@ class ModelsManagementClient:
         try:
             response: Final = session.send(request.prepare(), timeout=self._timeout)
             response.raise_for_status()
-            return response.json()
+            return _JSON_OBJECT.validate_python(response.json())
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 401:
                 raise UnauthorizedError(e)

litellm/proxy/common_request_processing.py

@@ -266,6 +266,8 @@ from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
     AnthropicStreamModelRestamper,
 )
 from litellm.proxy.litellm_pre_call_utils import (
+    ROUTER_SCHEDULED_ROUTE_TYPES,
+    SCHEDULER_PRIORITY_HEADER,
     LiteLLMProxyRequestSetup,
     add_litellm_data_to_request,
     get_metadata_variable_name,
@@ -2478,14 +2480,20 @@ class ProxyBaseLLMRequestProcessing:
                     self.data["model"] = alias_target
 
         aliased_model: Final = self.data.get("model")
+        if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
+            self.data.update(
+                llm_router.priority_header_update(
+                    model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
+                )
+            )
 
         litellm_call_id: Final = resolve_litellm_call_id(request.headers.get("x-litellm-call-id"))
         self.data["litellm_call_id"] = litellm_call_id
         bind_telemetry_call_id(litellm_call_id)
-        DDSpanTagger.tag_call_id(self.data.get("litellm_call_id"))
+        DDSpanTagger.tag_call_id(litellm_call_id)
         DDSpanTagger.tag_request(
             user_api_key_dict=user_api_key_dict,
-            requested_model=self.data.get("model"),
+            requested_model=aliased_model,
         )
 
         ### AUTO STREAM USAGE TRACKING ###

litellm/proxy/db/accounting_ledger.py

@@ -0,0 +1,269 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import AsyncGenerator, Callable
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias
+
+from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter
+from typing_extensions import assert_never
+
+from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent
+from litellm.proxy.db.db_span import db_span
+from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
+
+if TYPE_CHECKING:
+    from prisma import Prisma
+
+    from litellm.proxy.utils import PrismaClient
+
+AccountingEvent: TypeAlias = InboundUsageEvent | OutboundUsageEvent | CostEvent
+_JSON: Final = TypeAdapter(dict[str, JsonValue])
+_INSTANT: Final[TypeAdapter[datetime]] = TypeAdapter(AwareDatetime)
+_STRINGS: Final = TypeAdapter(tuple[dict[str, str], ...])
+_TIMEOUT: Final = timedelta(seconds=10)
+
+
+class AccountingConflictError(ValueError):
+    pass
+
+
+_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
+
+
+@dataclass(frozen=True, slots=True)
+class _AcceptedEvent:
+    event_id: str
+    kind: EventKind
+    source_event_id: str | None
+    occurred_at: datetime
+    payload: str
+    payload_hash: str
+    targets: tuple[str, ...]
+    schema_version: int
+    version: int
+
+
+def _json(value: object) -> str:
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
+
+
+def _prepare(
+    event: AccountingEvent,
+    targets: tuple[DeliveryTarget, ...],
+    envelope: str,
+    schema_version: int,
+    version: int,
+) -> _AcceptedEvent:
+    if type(schema_version) is not int or type(version) is not int or min(schema_version, version) < 1:
+        raise ValueError("Accounting versions must be positive integers")
+    validated: Final = type(event).model_validate_json(event.model_dump_json())
+    payload: Final = _JSON.validate_json(envelope)
+    if payload.get("event") != validated.model_dump(mode="json"):
+        raise ValueError("Accounting envelope must contain the matching public event")
+    destinations: Final = tuple(sorted(frozenset(DeliveryTarget(target).value for target in targets)))
+    serialized: Final = _json(payload)
+    identity: Final = _json((schema_version, version, destinations, payload))
+    kind: Final = (
+        EventKind.COST
+        if isinstance(validated, CostEvent)
+        else EventKind.INBOUND_USAGE
+        if isinstance(validated, InboundUsageEvent)
+        else EventKind.OUTBOUND_USAGE
+    )
+    return _AcceptedEvent(
+        event_id=validated.event_id,
+        kind=kind,
+        source_event_id=validated.usage_event_id if isinstance(validated, CostEvent) else None,
+        occurred_at=validated.occurred_at if isinstance(validated, CostEvent) else validated.ended_at,
+        payload=serialized,
+        payload_hash=hashlib.sha256(identity.encode()).hexdigest(),
+        targets=destinations,
+        schema_version=schema_version,
+        version=version,
+    )
+
+
+_INSERT_EVENT: Final = """
+INSERT INTO "LiteLLM_AccountingEvent"
+    (event_id, kind, source_event_id, schema_version, payload_hash, occurred_at, payload)
+VALUES ($1, $2::"AccountingEventKind", $3, $4::int, $5, $6::timestamp, $7::jsonb)
+ON CONFLICT (event_id) DO NOTHING RETURNING event_id
+"""
+_INSERT_DELIVERIES: Final = """
+INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
+SELECT $1, value::"AccountingDeliveryTarget", $3::int FROM jsonb_array_elements_text($2::jsonb)
+"""
+_LOCK_SOURCE: Final = """
+SELECT event_id FROM "LiteLLM_AccountingEvent"
+WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE
+"""
+_EXISTING_COST: Final = """
+SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_event_id = $1
+"""
+_LOCK_EVENT: Final = """
+SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
+"""
+_COMPLETED_FAMILY: Final = """
+root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
+AND root.recorded_at < $1::timestamp
+AND (root.kind = 'inbound_usage' OR EXISTS (
+    SELECT 1 FROM "LiteLLM_AccountingEvent" AS cost WHERE cost.source_event_id = root.event_id AND cost.kind = 'cost'
+))
+AND NOT EXISTS (
+    SELECT 1 FROM "LiteLLM_AccountingEvent" AS member
+    WHERE (member.event_id = root.event_id OR (member.kind = 'cost' AND member.source_event_id = root.event_id))
+    AND (member.recorded_at >= $1::timestamp
+        OR EXISTS (SELECT 1 FROM "LiteLLM_AccountingDelivery" AS delivery
+            WHERE delivery.event_id = member.event_id AND delivery.state != 'applied'))
+)
+"""
+_RETENTION_CANDIDATES: Final = (
+    'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE '
+    + _COMPLETED_FAMILY
+    + " ORDER BY root.recorded_at, root.event_id LIMIT $2::int FOR UPDATE OF root SKIP LOCKED"
+)
+_ELIGIBLE_GROUP: Final = (
+    'SELECT root.event_id FROM "LiteLLM_AccountingEvent" AS root WHERE root.event_id = $2 AND ' + _COMPLETED_FAMILY
+)
+_DELETE_DELIVERIES: Final = """
+DELETE FROM "LiteLLM_AccountingDelivery" WHERE event_id IN (
+    SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
+)
+"""
+_DELETE_EVENTS: Final = """
+DELETE FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 OR (kind = 'cost' AND source_event_id = $1)
+"""
+
+
+_Conflict: TypeAlias = Literal["missing_source", "competing_cost", "identity_conflict"]
+
+
+async def _accept(db: Prisma, record: _AcceptedEvent) -> Literal["inserted", "duplicate"] | _Conflict:
+    if record.source_event_id is not None:
+        source: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_SOURCE, record.source_event_id))
+        if not source:
+            return "missing_source"
+        costs: Final = _STRINGS.validate_python(tuple(await db.query_raw(_EXISTING_COST, record.source_event_id)))
+        if any(cost["event_id"] != record.event_id for cost in costs):
+            return "competing_cost"
+    inserted: Final = _STRINGS.validate_python(
+        await db.query_raw(
+            _INSERT_EVENT,
+            record.event_id,
+            record.kind.value,
+            record.source_event_id,
+            record.schema_version,
+            record.payload_hash,
+            record.occurred_at.astimezone(timezone.utc).isoformat(),
+            record.payload,
+        )
+    )
+    if not inserted:
+        existing: Final = _STRINGS.validate_python(tuple(await db.query_raw(_LOCK_EVENT, record.event_id)))
+        if not existing or existing[0]["payload_hash"] != record.payload_hash:
+            return "identity_conflict"
+        return "duplicate"
+    await db.execute_raw(_INSERT_DELIVERIES, record.event_id, _json(record.targets), record.version)
+    return "inserted"
+
+
+async def _delete_completed(db: Prisma, event_id: str, cutoff: str) -> int:
+    if not _STRINGS.validate_python(await db.query_raw(_ELIGIBLE_GROUP, cutoff, event_id)):
+        return 0
+    await db.execute_raw(_DELETE_DELIVERIES, event_id)
+    return await db.execute_raw(_DELETE_EVENTS, event_id)
+
+
+class AccountingLedger:
+    def __init__(self, transaction: Callable[[], _Transaction]) -> None:
+        self._transaction: Final = transaction
+
+    @classmethod
+    def for_client(cls, client: PrismaClient) -> AccountingLedger:
+        from prisma.client import TransactionManager
+
+        managers: Final = TypeAdapter(TransactionManager, config=ConfigDict(arbitrary_types_allowed=True))
+
+        @asynccontextmanager
+        async def transaction() -> AsyncGenerator[Prisma]:
+            primary: Final = writer_wrapper(client.db)
+            manager: Final = managers.validate_python(primary.tx(timeout=_TIMEOUT))
+            async with db_span("accounting_ledger", "LiteLLM_AccountingEvent"), manager as db:
+                await db.execute_raw("SET LOCAL statement_timeout = 5000")
+                await db.execute_raw("SET LOCAL lock_timeout = 1000")
+                yield db
+
+        return cls(transaction)
+
+    async def insert_event(
+        self,
+        event: AccountingEvent,
+        targets: tuple[DeliveryTarget, ...],
+        *,
+        envelope: str,
+        schema_version: int = 1,
+        version: int = 1,
+    ) -> Literal["inserted", "duplicate"]:
+        """Accept T5/T6's serialized envelope, containing its public event under `event`.
+
+        The immutable acceptance hash includes the target set and delivery version.
+        Changing either for an accepted event conflicts. T7 must provide a separate
+        add-target or new-version workflow that preserves receipts and prevents
+        duplicate contributions; redelivery cannot silently create or reset work.
+
+        The Prisma adapter opens one interactive transaction per event, including
+        two SET LOCAL statements and a source-row lock for costs. I/O is async,
+        but awaiting this method inline adds commit latency per event, potentially
+        several times per request. T5 owns bounded asynchronous capture and an
+        explicit batching strategy; looping over this method does not batch commits.
+        Success acknowledges commit, independently of HTTP response acknowledgement.
+        """
+        record: Final = _prepare(event, targets, envelope, schema_version, version)
+        async with self._transaction() as db:
+            result: Final = await _accept(db, record)
+        match result:
+            case "inserted" | "duplicate":
+                return result
+            case "missing_source":
+                raise AccountingConflictError("Cost source must be a durably accepted outbound usage event")
+            case "competing_cost":
+                raise AccountingConflictError("Usage event already has an authoritative cost")
+            case "identity_conflict":
+                raise AccountingConflictError("Event identity conflicts with its accepted content or delivery set")
+            case _:
+                assert_never(result)
+
+    async def purge_completed(
+        self,
+        *,
+        redelivery_before: datetime | None = None,
+        recovery_before: datetime | None = None,
+        limit: int = 100,
+    ) -> int:
+        """Delete at most `limit` completed families once both retention owners permit it.
+
+        T8 must supply a coherent recovery watermark including outstanding holds.
+        Omitting either watermark disables deletion. Each usage row and its cost
+        remain together so a retained charge never loses its source or vice versa.
+        Applied receipts are deleted with the family, ending its retained dedup
+        protection. The caller must guarantee events recorded before
+        redelivery_before cannot be redelivered through supported paths.
+        T8's recovery_before must also protect checkpoints,
+        outstanding holds and unfinished settlements; age alone is insufficient.
+        """
+        if type(limit) is not int or not 1 <= limit <= 1000:
+            raise ValueError("Accounting retention batch size must be between 1 and 1000")
+        if redelivery_before is None or recovery_before is None:
+            return 0
+        cutoff: Final = min(
+            _INSTANT.validate_python(redelivery_before).astimezone(timezone.utc),
+            _INSTANT.validate_python(recovery_before).astimezone(timezone.utc),
+        ).isoformat()
+        async with self._transaction() as db:
+            roots: Final = _STRINGS.validate_python(tuple(await db.query_raw(_RETENTION_CANDIDATES, cutoff, limit)))
+            deleted: Final = tuple([await _delete_completed(db, root["event_id"], cutoff) for root in roots])
+        return sum(deleted)

litellm/proxy/db/budget_window_spend_writer.py

@@ -64,6 +64,16 @@ _ROLL_WINDOW_SPEND_SQL: Final = (
     "AND window_start < ($4::timestamptz AT TIME ZONE 'UTC')"
 )
 
+_OPEN_WINDOW_SPEND_SQL: Final = (
+    'INSERT INTO "LiteLLM_BudgetWindowSpend" '
+    "(entity_type, entity_id, window_duration, window_start, spend, created_at, updated_at) "
+    "VALUES ($1, $2, $3, ($4::timestamptz AT TIME ZONE 'UTC'), 0, "
+    "($4::timestamptz AT TIME ZONE 'UTC'), ($4::timestamptz AT TIME ZONE 'UTC')) "
+    "ON CONFLICT (entity_type, entity_id, window_duration) DO UPDATE SET "
+    "window_start = EXCLUDED.window_start, spend = 0, updated_at = EXCLUDED.updated_at "
+    "WHERE \"LiteLLM_BudgetWindowSpend\".window_start <= ($5::timestamptz AT TIME ZONE 'UTC')"
+)
+
 _SEED_FROM_SPEND_LOGS_KEY_SQL: Final = (
     "SELECT COALESCE(SUM(spend), 0.0) AS total, "
     "COALESCE(SUM(spend) FILTER (WHERE \"startTime\" < ($3::timestamptz AT TIME ZONE 'UTC')), 0.0) AS before_batch "
@@ -389,3 +399,29 @@ async def roll_window_spend_row(
             to_naive_utc(new_window_start),
             to_naive_utc(datetime.now(timezone.utc)),
         )
+
+
+async def open_window_spend_row(
+    prisma_client: "PrismaClient",
+    entity_type: str,
+    entity_id: str,
+    window_duration: str,
+    window_start: datetime,
+    replaces_window_start: datetime | None,
+) -> None:
+    """Start a window at `window_start` unless another writer already started one.
+
+    The row is created when it does not exist, and moved onto the new window only
+    while it still holds `replaces_window_start`, the ended window the caller saw.
+    A writer that lost the race leaves the winner's window in place, so reading
+    the row back gives every writer the same window start.
+    """
+    async with db_span("open_window_spend_row", "LiteLLM_BudgetWindowSpend"):
+        await prisma_client.db.execute_raw(
+            _OPEN_WINDOW_SPEND_SQL,
+            entity_type,
+            entity_id,
+            window_duration,
+            to_naive_utc(window_start),
+            None if replaces_window_start is None else to_naive_utc(replaces_window_start),
+        )

litellm/proxy/guardrails/_content_utils.py

@@ -305,6 +305,11 @@ def iter_message_text(data: Mapping[str, object]) -> Iterator[str]:
         yield from _iter_text_parts_in_content(message.get("content"))
 
 
+def _is_tool_message(message: object) -> bool:
+    fields: Final = _text_object(message)
+    return fields is not None and str(fields.get("role") or "").lower() == "tool"
+
+
 def _call_item(item: object) -> tuple[str, object] | None:
     """Return ``(field, value)`` for a Responses tool-call input item, else None."""
     fields: Final = _text_object(item)
@@ -316,11 +321,14 @@ def walk_user_text(
     data: dict[str, object],
     visit: Callable[[str], str],
     visit_tool: Callable[[JsonValue], JsonValue] | None = None,
+    *,
+    skip_tool_message: bool = False,
 ) -> int:
     """Rewrite every text fragment in place via ``visit``.
 
     Mutates ``data["messages"]`` and ``data["input"]`` (including tool payloads), ``data["system"]`` and
-    ``data["instructions"]``. Returns the number
+    ``data["instructions"]``. ``skip_tool_message`` leaves ``role: tool`` messages and ``tool_result``
+    blocks as sent. Returns the number
     of fragments visited so callers can short-circuit when nothing was
     inspected.
     """
@@ -353,6 +361,8 @@ def walk_user_text(
 
     def _rewrite_part(part: Mapping[str, object], native: bool) -> Mapping[str, object]:
         nonlocal visited
+        if skip_tool_message and part.get("type") == "tool_result":
+            return part
         text: Final = _part_text(part)
         rewritten: Final = {**part, **({"text": visit(text)} if text is not None else {})}
         if text is not None:
@@ -384,18 +394,17 @@ def walk_user_text(
         return content
 
     messages: Final = data.get("messages")
-    if isinstance(messages, list):
-        for message in messages:
-            if not isinstance(message, dict):
-                continue
-            if "content" in message:
-                message["content"] = _rewrite_content(message["content"])
-            calls = message.get("tool_calls")
-            if isinstance(calls, list):
-                for call in calls:
-                    function = call.get("function") if isinstance(call, dict) else None
-                    if isinstance(function, dict) and "arguments" in function:
-                        function["arguments"] = _rewrite_arguments(function["arguments"])
+    for message in _object_list(messages) or ():
+        if (skip_tool_message and _is_tool_message(message)) or not isinstance(message, dict):
+            continue
+        if "content" in message:
+            message["content"] = _rewrite_content(message["content"])
+        calls = message.get("tool_calls")
+        if isinstance(calls, list):
+            for call in calls:
+                function = call.get("function") if isinstance(call, dict) else None
+                if isinstance(function, dict) and "arguments" in function:
+                    function["arguments"] = _rewrite_arguments(function["arguments"])
 
     if "system" in data:
         data["system"] = _rewrite_content(data["system"])  # rebind-ok: preserve the native hook's in-place contract

litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py

@@ -169,7 +169,7 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail):
 
         from litellm.types.llms.openai import OpenAIModerationResponse
 
-        return OpenAIModerationResponse(**response.json())
+        return OpenAIModerationResponse.model_validate(response.json())
 
     def _check_moderation_result(self, moderation_response: "OpenAIModerationResponse") -> None:
         """

litellm/proxy/hooks/model_budget_shared_windows.py

@@ -0,0 +1,176 @@
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
+
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.proxy.db.budget_window_spend_writer import open_window_spend_row
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
+    WindowSpendTransaction,
+    build_window_spend_transaction,
+)
+from litellm.repositories.table_repositories import BudgetWindowSpendRepository
+from litellm.router_strategy.budget_limiter import budget_window_seconds
+
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
+MODEL_BUDGET_WINDOW_ENTITY_TYPE: Final = "model_max_budget"
+_READ_TTL_SECONDS: Final = 5
+
+
+@dataclass(frozen=True, slots=True)
+class SharedWindow:
+    started_at: float
+    spend: float
+
+    def is_open(self, budget_duration: str, now: float) -> bool:
+        return now < self.started_at + budget_window_seconds(self.started_at, budget_duration)
+
+
+class LocalTTLCache(Protocol):
+    def get_cache(self, key: str) -> object: ...
+
+    def set_cache(self, key: str, value: object, *, ttl: float) -> None: ...
+
+    def delete_cache(self, key: str) -> None: ...
+
+
+class SharedWindows(Protocol):
+    async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None: ...
+
+    async def open(self, spend_key: str, budget_duration: str, now: float) -> float | None: ...
+
+    async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None: ...
+
+
+PrismaClientSource: TypeAlias = Callable[[], "PrismaClient | None"]
+WindowSpendSink: TypeAlias = Callable[[WindowSpendTransaction], Awaitable[None]]
+
+
+def _proxy_prisma_client() -> "PrismaClient | None":
+    try:
+        from litellm.proxy.proxy_server import prisma_client
+    except ImportError:
+        return None
+    return prisma_client
+
+
+async def _queue_on_proxy_spend_writer(transaction: WindowSpendTransaction) -> None:
+    from litellm.proxy.proxy_server import proxy_logging_obj
+
+    await proxy_logging_obj.db_spend_update_writer.window_spend_update_queue.add_update(transaction)
+
+
+@dataclass(frozen=True, slots=True)
+class _Reading:
+    window: SharedWindow | None
+
+
+class DatabaseSharedWindows:
+    def __init__(
+        self,
+        prisma_client_source: PrismaClientSource = _proxy_prisma_client,
+        sink: WindowSpendSink = _queue_on_proxy_spend_writer,
+        read_ttl_seconds: int = _READ_TTL_SECONDS,
+    ) -> None:
+        self._prisma_client_source = prisma_client_source
+        self._sink = sink
+        self._read_ttl_seconds = read_ttl_seconds
+        self._readings: LocalTTLCache = InMemoryCache(max_size_in_memory=10_000, default_ttl=read_ttl_seconds)
+
+    async def read(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+        cached: Final = self._readings.get_cache(key=spend_key)
+        if isinstance(cached, _Reading):
+            return cached.window
+        window: Final = await self._read_row(spend_key=spend_key, budget_duration=budget_duration)
+        self._readings.set_cache(key=spend_key, value=_Reading(window=window), ttl=self._read_ttl_seconds)
+        return window
+
+    async def open(self, spend_key: str, budget_duration: str, now: float) -> float | None:
+        """When the window every replica shares started, opening one at `now` if none is open.
+
+        Without a database every window is this replica's own, so it starts at `now`. None means the database did
+        not answer, so this replica cannot know which window the others are in yet.
+        """
+        prisma_client: Final = self._prisma_client_source()
+        if prisma_client is None:
+            return now
+        try:
+            seen: Final = await _fetch_row(
+                prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
+            )
+            if seen is not None and seen.is_open(budget_duration=budget_duration, now=now):
+                return seen.started_at
+            await open_window_spend_row(
+                prisma_client=prisma_client,
+                entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+                entity_id=spend_key,
+                window_duration=budget_duration,
+                window_start=datetime.fromtimestamp(now, tz=timezone.utc),
+                replaces_window_start=None
+                if seen is None
+                else datetime.fromtimestamp(seen.started_at, tz=timezone.utc),
+            )
+            opened: Final = await _fetch_row(
+                prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=True
+            )
+        except Exception:  # noqa: BLE001  # an unreachable database leaves this replica on its own window
+            verbose_proxy_logger.exception("Unable to open the shared model budget window")
+            return None
+        if opened is None or not opened.is_open(budget_duration=budget_duration, now=now):
+            return None
+        return opened.started_at
+
+    async def add(self, spend_key: str, budget_duration: str, started_at: float, spend: float) -> None:
+        if self._prisma_client_source() is None:
+            return
+        try:
+            await self._sink(
+                build_window_spend_transaction(
+                    entity_type=MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+                    entity_id=spend_key,
+                    window_duration=budget_duration,
+                    window_start=datetime.fromtimestamp(started_at, tz=timezone.utc),
+                    spend=spend,
+                )
+            )
+        except Exception:  # noqa: BLE001  # spend tracking must never fail the success callback
+            verbose_proxy_logger.debug("Unable to queue shared model budget spend", exc_info=True)
+
+    async def _read_row(self, spend_key: str, budget_duration: str) -> SharedWindow | None:
+        prisma_client: Final = self._prisma_client_source()
+        if prisma_client is None:
+            return None
+        try:
+            return await _fetch_row(
+                prisma_client=prisma_client, spend_key=spend_key, budget_duration=budget_duration, use_writer=False
+            )
+        except Exception:  # noqa: BLE001  # an unreadable row leaves enforcement on this replica's own counter
+            verbose_proxy_logger.exception("Unable to read shared model budget spend")
+            return None
+
+
+async def _fetch_row(
+    prisma_client: "PrismaClient", spend_key: str, budget_duration: str, use_writer: bool
+) -> SharedWindow | None:
+    row: Final = await BudgetWindowSpendRepository(prisma_client, use_writer=use_writer).table.find_unique(
+        where={
+            "entity_type_entity_id_window_duration": {
+                "entity_type": MODEL_BUDGET_WINDOW_ENTITY_TYPE,
+                "entity_id": spend_key,
+                "window_duration": budget_duration,
+            }
+        }
+    )
+    if row is None:
+        return None
+    window_start: Final = row.window_start
+    started_at: Final = (
+        window_start if window_start.tzinfo is not None else window_start.replace(tzinfo=timezone.utc)
+    ).timestamp()
+    return SharedWindow(started_at=started_at, spend=float(row.spend or 0.0))
+
+
+PROXY_SHARED_WINDOWS: Final = DatabaseSharedWindows()

litellm/proxy/litellm_pre_call_utils.py

@@ -118,6 +118,9 @@ _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
 
 _SHA256_HEX_RE: Final = re.compile(r"^[0-9a-f]{64}$")
 
+SCHEDULER_PRIORITY_HEADER: Final = "x-litellm-priority"
+ROUTER_SCHEDULED_ROUTE_TYPES: Final = frozenset({"acompletion", "atext_completion"})
+
 # W3C Trace Context traceparent header: https://www.w3.org/TR/trace-context/
 # e.g. "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
 _TRACEPARENT_RE: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-[0-9a-f]{16}-[0-9a-f]{2}$", re.IGNORECASE)
@@ -688,7 +691,7 @@ def _metadata_string_tags(metadata_objects: Iterable[Mapping[str, object]]) -> t
 
 
 def _strip_untrusted_request_header_controls(
-    headers: Any,
+    headers: object,
     *,
     allow_client_message_redaction_opt_out: bool = False,
 ) -> None:
@@ -1692,7 +1695,7 @@ class LiteLLMProxyRequestSetup:
 
     @staticmethod
     def add_internal_user_from_user_mapping(
-        general_settings: dict | None,
+        general_settings: Mapping[str, object] | None,
         user_api_key_dict: UserAPIKeyAuth,
         headers: dict,
     ) -> UserAPIKeyAuth:
@@ -1711,7 +1714,7 @@ class LiteLLMProxyRequestSetup:
         return user_api_key_dict
 
     @staticmethod
-    def get_user_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
+    def get_user_from_headers(headers: dict, general_settings: Mapping[str, object] | None = None) -> str | None:
         """
         Get the user from the specified header if `general_settings.user_header_name` is set.
         """
@@ -1732,7 +1735,9 @@ class LiteLLMProxyRequestSetup:
         return user
 
     @staticmethod
-    def get_openai_org_id_from_headers(headers: dict, general_settings: dict | None = None) -> str | None:
+    def get_openai_org_id_from_headers(
+        headers: dict, general_settings: Mapping[str, object] | None = None
+    ) -> str | None:
         """
         Get the OpenAI Org ID from the headers.
         """
@@ -1823,7 +1828,7 @@ class LiteLLMProxyRequestSetup:
         headers: dict,
         request_data: Mapping[str, object],
         user_api_key_dict: UserAPIKeyAuth,
-        general_settings: dict[str, object] | None = None,
+        general_settings: Mapping[str, object] | None = None,
     ) -> LitellmDataForBackendLLMCall:
         """
         - Adds user from headers

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -231,9 +231,12 @@ def _shadow_eval_attempts(prisma_client: "PrismaClient") -> _ShadowEvalAttemptTa
     return _ShadowEvalAttemptRepository(prisma_client).table
 
 
+_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
+
+
 async def _query_raw(prisma_client: "PrismaClient", query: str, *args: object) -> Sequence[Mapping[str, object]]:
     async with db_span("auto_router_report_query", sql_relation(query)):
-        return await prisma_client.db.query_raw(query, *args)
+        return _RAW_ROWS.validate_python(await prisma_client.db.query_raw(query, *args))
 
 
 async def _execute_raw(prisma_client: "PrismaClient", query: str, *args: object) -> int:

litellm/proxy/management_endpoints/common_daily_activity.py

@@ -3,6 +3,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence
 from collections.abc import Set as AbstractSet
 from dataclasses import dataclass, replace
 from datetime import date, datetime, timedelta
+from itertools import groupby
 from types import MappingProxyType
 from typing import Final, Literal, NoReturn, Protocol
 
@@ -28,6 +29,10 @@ from litellm.types.proxy.management_endpoints.common_daily_activity import (
     DailySpendData,
     DailySpendMetadata,
     GroupedData,
+    GuardrailKeyMetric,
+    GuardrailMetadata,
+    GuardrailMetric,
+    GuardrailSpendMetrics,
     KeyMetadata,
     KeyMetricWithMetadata,
     MetricWithMetadata,
@@ -39,6 +44,7 @@ from litellm.types.repositories.daily_activity import (
     DailyActivityTable,
     EntityRollupRow,
     GroupingSetsRow,
+    GuardrailSpendRow,
     KeyMetadataRow,
     RollupMetricsRow,
     SpendLogsWindow,
@@ -1013,17 +1019,68 @@ def _fold_entity_rollups_sync(
             )
 
 
+def _guardrail_metric(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> GuardrailMetric:
+    return GuardrailMetric(
+        metrics=GuardrailSpendMetrics(
+            spend=sum(row.spend for row in rows),
+            untracked_units=sum(row.untracked_units for row in rows),
+        ),
+        metadata=GuardrailMetadata(guardrail_name=rows[0].guardrail_name),
+        api_key_breakdown={
+            row.api_key: GuardrailKeyMetric(
+                metrics=GuardrailSpendMetrics(spend=row.spend, untracked_units=row.untracked_units),
+                metadata=day.breakdown.api_keys[row.api_key].metadata,
+            )
+            for row in rows
+            if row.api_key in day.breakdown.api_keys
+        },
+    )
+
+
+def _day_with_guardrails(day: DailySpendData, rows: Sequence[GuardrailSpendRow]) -> DailySpendData:
+    by_guardrail: Final = groupby(sorted(rows, key=lambda row: row.guardrail_id), key=lambda row: row.guardrail_id)
+    guardrails: Final = {guardrail_id: _guardrail_metric(day, tuple(group)) for guardrail_id, group in by_guardrail}
+    return day.model_copy(update={"breakdown": day.breakdown.model_copy(update={"guardrails": guardrails})})
+
+
+def _with_guardrail_breakdown(
+    results: Sequence[DailySpendData], rows: Sequence[GuardrailSpendRow]
+) -> tuple[DailySpendData, ...]:
+    rows_by_date: Final = MappingProxyType(
+        {day: tuple(group) for day, group in groupby(sorted(rows, key=lambda row: row.date), key=lambda row: row.date)}
+    )
+    return tuple(_day_with_guardrails(day, rows_by_date.get(day.date.strftime("%Y-%m-%d"), ())) for day in results)
+
+
+async def _guardrail_spend_or_empty(
+    repository: DailyActivityRepository, scope: DailyActivityScope
+) -> tuple[GuardrailSpendRow, ...]:
+    from prisma.errors import PrismaError
+
+    try:
+        return await repository.guardrail_spend(scope)
+    except PrismaError as exc:
+        verbose_proxy_logger.warning("Returning daily activity without its guardrail breakdown: %s", exc)
+        return ()
+
+
 async def get_daily_activity_aggregated(
     repository: DailyActivityRepository,
     scope: DailyActivityScope,
     *,
     entity_metadata_field: Mapping[str, dict[str, object]] | None = None,
     include_entity_breakdown: bool = False,
+    include_guardrail_breakdown: bool = False,
     api_key_limit: int = constants.USAGE_TOP_API_KEYS_DEFAULT,
 ) -> SpendAnalyticsPaginatedResponse:
     try:
-        aggregated_rows: Final = await repository.aggregated(
-            scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
+        aggregated_rows, guardrail_rows = await asyncio.gather(
+            repository.aggregated(
+                scope, include_entity_breakdown=include_entity_breakdown, api_key_limit=api_key_limit
+            ),
+            _guardrail_spend_or_empty(repository, scope)
+            if include_guardrail_breakdown and not scope.model
+            else asyncio.sleep(0, result=()),
         )
         records: Final = aggregated_rows.grouping_rows
         aggregated: Final = await _aggregate_grouping_sets_records(
@@ -1058,8 +1115,13 @@ async def get_daily_activity_aggregated(
                 api_key_metadata=entity_key_metadata,
                 entity_metadata_field=entity_metadata_field,
             )
+        results: Final = (
+            list(await asyncio.to_thread(_with_guardrail_breakdown, aggregated["results"], guardrail_rows))
+            if guardrail_rows
+            else aggregated["results"]
+        )
         return SpendAnalyticsPaginatedResponse(
-            results=aggregated["results"],
+            results=results,
             metadata=DailySpendMetadata(
                 total_spend=aggregated["totals"].spend,
                 total_flat_cost=aggregated["totals"].flat_cost,

litellm/proxy/management_endpoints/daily_activity_routes.py

@@ -300,6 +300,7 @@ def _register_aggregated_route(router: APIRouter, resolver: EntityScopeResolver,
                 resolved.scope,
                 entity_metadata_field=resolved.entity_metadata,
                 include_entity_breakdown=resolver.include_entity_breakdown,
+                include_guardrail_breakdown=resolver.include_guardrail_breakdown,
                 api_key_limit=api_key_limit,
             )
             if resolver.entity != "team":

litellm/proxy/management_endpoints/daily_activity_scopes.py

@@ -56,6 +56,7 @@ class EntityScopeResolver:
     resolve: EntityScopeResolve
     include_entity_breakdown: bool
     operation_names: OperationNames
+    include_guardrail_breakdown: bool = False
 
 
 def _query_ids(value: str | None) -> tuple[str, ...] | None:
@@ -364,6 +365,7 @@ USER_RESOLVER = EntityScopeResolver(
     query=_user_query,
     resolve=_resolve_user,
     include_entity_breakdown=False,
+    include_guardrail_breakdown=True,
     operation_names=MappingProxyType(
         {
             "aggregated": "get_user_daily_activity_aggregated",
@@ -385,6 +387,7 @@ TEAM_RESOLVER = EntityScopeResolver(
     query=_team_query,
     resolve=_resolve_team,
     include_entity_breakdown=True,
+    include_guardrail_breakdown=True,
     operation_names=MappingProxyType(
         {
             "aggregated": "get_team_daily_activity_aggregated",
@@ -423,6 +426,7 @@ ORGANIZATION_RESOLVER = EntityScopeResolver(
     query=_organization_query,
     resolve=_resolve_organization,
     include_entity_breakdown=True,
+    include_guardrail_breakdown=True,
     operation_names=MappingProxyType(
         {
             "aggregated": "get_organization_daily_activity_aggregated",

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -499,7 +499,7 @@ async def new_user(
     - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
     - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
     - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
-    - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
+    - model_max_budget: Optional[dict] - Model-specific max budget for user, keyed by model name, `*` pattern, or model access group. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
     - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user.
@@ -1702,7 +1702,7 @@ async def user_update(
         - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
         - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
         - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
-        - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
+        - model_max_budget: Optional[dict] - Model-specific max budget for user, keyed by model name, `*` pattern, or model access group. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
         - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
         - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
         - mcp_rpm_limit: Optional[dict] - Per-MCP-server rpm limit, keyed by MCP server name {"github": 100, "slack": 200}. Enforced for keys and teams only; values set on a user are stored but not enforced per user.

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -2328,7 +2328,7 @@ async def generate_key_fn(
     - throttle_on_budget_exceeded: Optional[bool] - When the key exceeds its max_budget, throttle its tpm/rpm to the global budget_exceeded_throttle_percentage instead of blocking the key entirely.
     - enable_prompt_caching: Optional[bool] - Auto-inject prompt caching breakpoints (Anthropic cache_control markers) on requests made with this key. Supported Claude models on Anthropic, Bedrock, Vertex AI, and Azure AI only.
     - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
-    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
+    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
     - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
@@ -2538,7 +2538,7 @@ async def generate_service_account_key_fn(
     - metadata: Optional[dict] - Metadata for key, store information for key. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
     - guardrails: Optional[List[str]] - List of active guardrails for the key
     - permissions: Optional[dict] - key-specific permissions. Currently just used for turning off pii masking (if connected). Example - {"pii": false}
-    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
+    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}}. IF null or {} then no model specific budget.
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - model_rpm_limit: Optional[dict] - key-specific model rpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific rpm limit.
     - model_tpm_limit: Optional[dict] - key-specific model tpm limit. Example - {"text-davinci-002": 1000, "gpt-3.5-turbo": 1000}. IF null or {} then no model specific tpm limit.
@@ -3855,7 +3855,7 @@ async def update_key_fn(
     - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests)
     - spend: Optional[float] - Amount spent by key
     - max_budget: Optional[float] - Max budget for key
-    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
+    - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
     - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
     - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
     - soft_budget: Optional[float] - Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached. Set to null to remove the soft budget.
@@ -6303,7 +6303,7 @@ async def regenerate_key_fn(
         - tags: Optional[List[str]] - Tags for organizing keys (Enterprise only)
         - spend: Optional[float] - Amount spent by key
         - max_budget: Optional[float] - Max budget for key
-        - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
+        - model_max_budget: Optional[Dict[str, BudgetConfig]] - Model-specific budgets keyed by model name, `*` pattern like "claude-opus*", or model access group, e.g. {"gpt-4": {"budget_limit": 0.0005, "time_period": "30d"}}
         - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
         - budget_duration: Optional[str] - Budget reset period ("30d", "1h", etc.)
         - soft_budget: Optional[float] - Soft budget limit (warning vs. hard stop). Will trigger a slack alert when this soft budget is reached.

litellm/proxy/management_endpoints/router_settings_endpoints.py

@@ -10,7 +10,7 @@ GET /router/fields - Get router settings field definitions without values (for U
 import inspect
 from collections.abc import Mapping
 from types import MappingProxyType
-from typing import Any, Final, get_args
+from typing import Final, get_args
 
 from fastapi import APIRouter, Depends
 from pydantic import Field
@@ -32,7 +32,7 @@ router: Final = APIRouter()
 
 class RouterSettingsResponse(LiteLLMBaseModel):
     fields: list[RouterSettingsField] = Field(description="List of all configurable router settings with metadata")
-    current_values: dict[str, Any] = Field(description="Current values of router settings")
+    current_values: dict[str, object] = Field(description="Current values of router settings")
     routing_strategy_descriptions: dict[str, str] = Field(description="Descriptions for each routing strategy option")
     routing_group_strategies: tuple[str, ...] = Field(
         description="Strategies supported when constructing a routing group"

litellm/proxy/management_endpoints/team_endpoints.py

@@ -11,6 +11,7 @@ All /team management endpoints
 
 import asyncio
 import copy
+import itertools
 import json
 import math
 import traceback
@@ -139,7 +140,12 @@ from litellm.proxy.common_utils.user_api_key_cache import (
 from litellm.proxy.db.db_span import db_span
 from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
 from litellm.proxy.hooks.model_max_budget_limiter import (
+    RoutedModels,
     build_model_max_budget_usage,
+    entries_outranked_by,
+    entry_names_model,
+    is_enforced_budget_entry,
+    proxy_routed_models,
     resolve_model_budget,
 )
 from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamRole, is_team_admin, team_access_denied
@@ -1467,8 +1473,10 @@ def _check_team_model_budget_update_authority(
     data: UpdateTeamRequest,
     user_api_key_dict: UserAPIKeyAuth,
     existing_model_max_budget: Mapping[str, object] | None,
+    routed_models_source: Callable[[str | None], RoutedModels] = proxy_routed_models,
 ) -> None:
-    """Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap."""
+    """Like `_check_team_budget_update_authority`: only a proxy admin may raise, re-window or drop a per-model cap, or
+    add an entry that takes precedence over one."""
     if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
         return
     if "model_max_budget" not in data.model_fields_set or not existing_model_max_budget:
@@ -1487,29 +1495,97 @@ def _check_team_model_budget_update_authority(
                 )
             },
         )
-    for model_name, proposed in requested.items():
-        governing = resolve_model_budget(model=model_name, model_max_budget=existing_model_max_budget)
-        if governing is None:
-            continue
-        cap = governing.budget_config
-        if cap.max_budget is None:
-            continue
-        if (
-            proposed.max_budget is None
-            or proposed.max_budget > cap.max_budget
-            or proposed.budget_duration != cap.budget_duration
-        ):
-            raise HTTPException(
-                status_code=403,
-                detail={
-                    "error": (
-                        f"Only a proxy admin can raise a team's model_max_budget for {model_name!r} or change its "
-                        f"budget_duration. Current max_budget={cap.max_budget} per {cap.budget_duration} "
-                        f"(entry {governing.budget_model!r}), requested={proposed.max_budget} per "
-                        f"{proposed.budget_duration}."
-                    )
-                },
+    routed_models: Final = routed_models_source(data.team_id)
+    for entry, proposed in requested.items():
+        existing = _existing_model_cap(existing_model_max_budget.get(entry))
+        if existing is not None:
+            _check_existing_model_cap_kept(entry=entry, cap_entry=entry, proposed=proposed, existing=existing)
+            if is_enforced_budget_entry(existing) or not is_enforced_budget_entry(proposed):
+                continue
+        governing = resolve_model_budget(model=entry, model_max_budget=existing_model_max_budget, team_id=data.team_id)
+        if governing is not None and entry_names_model(entry=governing.budget_model, model=entry):
+            _check_existing_model_cap_kept(
+                entry=entry, cap_entry=governing.budget_model, proposed=proposed, existing=governing.budget_config
             )
+        _check_new_entry_outranks_no_cap(
+            entry=entry,
+            existing_model_max_budget=existing_model_max_budget,
+            routed_models=routed_models,
+            team_id=data.team_id,
+        )
+
+
+def _check_existing_model_cap_kept(entry: str, cap_entry: str, proposed: BudgetConfig, existing: BudgetConfig) -> None:
+    if existing.max_budget is None or not _looser_model_cap(proposed=proposed, cap=existing):
+        return
+    raise HTTPException(
+        status_code=403,
+        detail={
+            "error": (
+                f"Only a proxy admin can raise a team's model_max_budget for {entry!r} or change its "
+                f"budget_duration. Current max_budget={existing.max_budget} per {existing.budget_duration} "
+                f"(entry {cap_entry!r}), requested={proposed.max_budget} per {proposed.budget_duration}."
+            )
+        },
+    )
+
+
+def _check_new_entry_outranks_no_cap(
+    entry: str,
+    existing_model_max_budget: Mapping[str, object],
+    routed_models: RoutedModels,
+    team_id: str | None,
+) -> None:
+    """A new entry gets its own counter, so the models it takes from a capped entry stop drawing from that cap."""
+    outranked: Final = next(
+        _capped_entries_outranked_by(
+            entry=entry,
+            existing_model_max_budget=existing_model_max_budget,
+            routed_models=routed_models,
+            team_id=team_id,
+        ),
+        None,
+    )
+    if outranked is None:
+        return
+    outranked_name, cap = outranked
+    raise HTTPException(
+        status_code=403,
+        detail={
+            "error": (
+                f"Only a proxy admin can add a team model_max_budget entry {entry!r} that takes precedence over "
+                f"{outranked_name!r}, since the models they share would stop drawing from its max_budget="
+                f"{cap.max_budget} per {cap.budget_duration}."
+            )
+        },
+    )
+
+
+def _capped_entries_outranked_by(
+    entry: str,
+    existing_model_max_budget: Mapping[str, object],
+    routed_models: RoutedModels,
+    team_id: str | None,
+) -> Iterator[tuple[str, BudgetConfig]]:
+    governing: Final = resolve_model_budget(model=entry, model_max_budget=existing_model_max_budget, team_id=team_id)
+    outranked_names: Final = itertools.chain(
+        ()
+        if governing is None or entry_names_model(entry=governing.budget_model, model=entry)
+        else (governing.budget_model,),
+        entries_outranked_by(entry=entry, model_max_budget=existing_model_max_budget, routed_models=routed_models),
+    )
+    outranked_caps: Final = ((name, _existing_model_cap(existing_model_max_budget[name])) for name in outranked_names)
+    yield from (
+        (name, cap) for name, cap in outranked_caps if name != entry and cap is not None and cap.max_budget is not None
+    )
+
+
+def _looser_model_cap(proposed: BudgetConfig, cap: BudgetConfig) -> bool:
+    return (
+        proposed.max_budget is None
+        or (cap.max_budget is not None and proposed.max_budget > cap.max_budget)
+        or (is_enforced_budget_entry(cap) and proposed.budget_duration != cap.budget_duration)
+    )
 
 
 def _should_auto_add_team_creator(
@@ -1572,7 +1648,7 @@ async def new_team(
     - prompts: Optional[List[str]] - List of prompts that the team is allowed to use.
     - organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`.
     - model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias)
-    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
+    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model, keyed by model name, `*` pattern, or model access group. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
     - guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails)
     - policies: Optional[List[str]] - Policies for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies)
     - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the team. Proxy admin only.
@@ -2350,7 +2426,7 @@ async def update_team(
     - tags: Optional[List[str]] - Tags for [tracking spend](https://litellm.vercel.app/docs/proxy/enterprise#tracking-spend-for-custom-tags) and/or doing [tag-based routing](https://litellm.vercel.app/docs/proxy/tag_routing).
     - organization_id: Optional[str] - The organization id of the team. Default is None. Create via `/organization/new`.
     - model_aliases: Optional[dict] - Model aliases for the team. [Docs](https://docs.litellm.ai/docs/proxy/team_based_routing#create-team-with-model-alias)
-    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
+    - model_max_budget: Optional[dict] - Per-model max budget every key on the team inherits unless the key sets its own for that model, keyed by model name, `*` pattern, or model access group. Example: {"gpt-4o": {"max_budget": 10, "budget_duration": "1d"}}
     - guardrails: Optional[List[str]] - Guardrails for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails)
     - policies: Optional[List[str]] - Policies for the team. [Docs](https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies)
     - disable_global_guardrails: Optional[bool] - Whether to disable global guardrails for the team. Proxy admin only.

litellm/proxy/management_helpers/team_roster_sync.py

@@ -40,6 +40,7 @@ from litellm.proxy.management_helpers.access_group_team_sync import (
 )
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
+from litellm.repositories.team_repository import TeamRepository
 
 if TYPE_CHECKING:
     from prisma import Prisma
@@ -144,7 +145,7 @@ def _changes_nothing(plan: RosterPlan) -> bool:
 
 
 async def _unchanged_roster(prisma_client: PrismaClient, team_id: str) -> RosterSync | TeamGone:
-    team_row: Final = await prisma_client.writer_db.litellm_teamtable.find_unique(where={"team_id": team_id})
+    team_row: Final = await TeamRepository(prisma_client, use_writer=True).table.find_unique(where={"team_id": team_id})
     if team_row is None:
         return TeamGone(team_id=team_id)
     team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())

litellm/proxy/mcp_server/discoverable_endpoints.py

@@ -300,12 +300,13 @@ def decode_state_hash(encrypted_state: str) -> dict:
     if decrypted_json is None:
         raise ValueError("Failed to decrypt state parameter")
 
-    state_data: Final = json.loads(decrypted_json)
+    state_data: Final = _DECODED_STATE.validate_python(json.loads(decrypted_json))
     return state_data
 
 
 _MANAGED_AUTH_CODE_PREFIX: Final = "llm_scope_"
 _DECODED_JSON: Final = TypeAdapter(object)
+_DECODED_STATE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
 
 
 class _ManagedAuthorizationCode(LiteLLMBaseModel):

litellm/proxy/openai_files_endpoints/files_endpoints.py

@@ -6,9 +6,12 @@
 ######################################################################
 
 import asyncio
+import functools
+import itertools
 import traceback
-from collections.abc import Mapping, Sequence
-from typing import Any, BinaryIO, Final, TypedDict, cast, get_args
+from collections.abc import Iterable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, BinaryIO, Final, NoReturn, TypedDict, cast, get_args
 
 import httpx
 from fastapi import (
@@ -28,12 +31,13 @@ from typing_extensions import ReadOnly
 import litellm
 from litellm import CreateFileRequest, get_secret_str
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import ANTHROPIC_DEFAULT_FILE_LIST_LIMIT
 from litellm.litellm_core_utils.cloud_storage_security import (
     is_managed_cloud_storage_uri,
 )
 from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
 from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME
-from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
 from litellm.llms.base_llm.managed_resources.isolation import build_list_page
 from litellm.proxy._types import *
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@@ -112,6 +116,9 @@ from litellm.types.llms.openai import (
     OpenAIFileObject,
     OpenAIFilesPurpose,
 )
+from litellm.types.proxy.auth.user_api_key_auth import user_api_key_has_admin_view
+from litellm.types.router import Deployment
+from litellm.types.utils import LlmProviders
 
 router: Final = APIRouter()
 
@@ -1673,6 +1680,185 @@ def _as_file_list_page(response: object) -> object:
     return FileListPage.model_validate(page)
 
 
+async def _list_owned_managed_files(
+    managed_files_obj: BaseFileEndpoints,
+    user_api_key_dict: UserAPIKeyAuth,
+    purpose: str | None,
+    limit: int | None,
+    after: str | None,
+    before: str | None,
+    model_scope: FileListModelScope | None,
+) -> FileListPage:
+    return await managed_files_obj.afile_list(
+        purpose=purpose,
+        litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+        user_api_key_dict=user_api_key_dict,
+        limit=limit,
+        after=after,
+        before=before,
+        model_scope=model_scope,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class _UnscopedFileList:
+    status_code: int
+    detail: str
+
+
+@dataclass(frozen=True, slots=True)
+class _ListedFiles:
+    model_scope: FileListModelScope | None
+    custom_llm_provider: str | None
+
+
+def _raise_unscoped_file_list(failure: _UnscopedFileList) -> NoReturn:
+    raise HTTPException(status_code=failure.status_code, detail=failure.detail)
+
+
+def _single_target_model_name(target_model_names: str) -> str | _UnscopedFileList:
+    target_model_names_list: Final = target_model_names.split(",")
+    if len(target_model_names_list) != 1:
+        return _UnscopedFileList(
+            status_code=400,
+            detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
+        )
+    return target_model_names_list[0]
+
+
+async def _requested_custom_llm_provider(request: Request, provider: str | None) -> str | None:
+    return (
+        provider
+        or get_custom_llm_provider_from_request_headers(request=request)
+        or get_custom_llm_provider_from_request_query(request=request)
+        or await get_custom_llm_provider_from_request_body(request=request)
+    )
+
+
+def _default_file_list_limit(custom_llm_provider: str | None) -> int | None:
+    return ANTHROPIC_DEFAULT_FILE_LIST_LIMIT if custom_llm_provider == LlmProviders.ANTHROPIC.value else None
+
+
+def _deployment_model_mapping_keys(deployment: Deployment) -> frozenset[str]:
+    return frozenset(key for key in (deployment.model_name, deployment.model_info.id) if key is not None)
+
+
+def _model_mapping_keys(deployments: Iterable[Deployment]) -> frozenset[str]:
+    return frozenset(itertools.chain.from_iterable(map(_deployment_model_mapping_keys, deployments)))
+
+
+def _deployments_serving_model(llm_router: Router, model: str) -> tuple[Deployment, ...]:
+    deployment: Final = llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) else None
+    if deployment is not None:
+        return (deployment,)
+    return tuple(Deployment.model_validate(listed) for listed in llm_router.get_model_list(model_name=model) or ())
+
+
+def _resolved_deployment_provider(deployment: Deployment) -> str | None:
+    try:
+        return litellm.get_llm_provider(
+            model=deployment.litellm_params.model,
+            custom_llm_provider=deployment.litellm_params.custom_llm_provider,
+        )[1]
+    except litellm.BadRequestError:
+        return None
+
+
+def _deployments_on_provider(llm_router: Router | None, custom_llm_provider: str) -> Iterator[Deployment]:
+    if llm_router is None:
+        return iter(())
+    return (
+        deployment
+        for deployment_id in llm_router.get_model_ids()
+        if (deployment := llm_router.get_deployment(model_id=deployment_id)) is not None
+        and _resolved_deployment_provider(deployment) == custom_llm_provider
+    )
+
+
+def _routed_model_name_on_provider(
+    llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
+) -> bool:
+    if model_name in llm_router.model_names:
+        return model_name in provider_model_names
+    return llm_router.pattern_router.routes_to_provider(model_name, custom_llm_provider)
+
+
+def _stored_model_name_routes_to_provider(
+    llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
+) -> bool:
+    routing_group: Final = llm_router.get_routing_group(model_name)
+    routed_model_names: Final = (
+        tuple(routing_group.models) if routing_group is not None else (llm_router.routable_model_group(model_name),)
+    )
+    return any(
+        _routed_model_name_on_provider(llm_router, custom_llm_provider, provider_model_names, routed_model_name)
+        for routed_model_name in routed_model_names
+    )
+
+
+def _files_on_provider(llm_router: Router | None, custom_llm_provider: str) -> _ListedFiles | _UnscopedFileList:
+    from litellm.proxy.pass_through_endpoints.managed_id_rewriter import passthrough_file_model_id
+
+    if custom_llm_provider not in litellm.provider_list:
+        return _UnscopedFileList(status_code=400, detail=f"Unknown provider '{custom_llm_provider}' for listing files.")
+    passthrough_model_id: Final = passthrough_file_model_id(custom_llm_provider)
+    deployments: Final = tuple(_deployments_on_provider(llm_router, custom_llm_provider))
+    model_mapping_keys: Final = _model_mapping_keys(deployments) | frozenset(
+        (passthrough_model_id,) if passthrough_model_id is not None else ()
+    )
+    if llm_router is None:
+        return _ListedFiles(FileListModelScope(model_mapping_keys), custom_llm_provider)
+    return _ListedFiles(
+        FileListModelScope(
+            model_mapping_keys,
+            routes_stored_model_name=functools.partial(
+                _stored_model_name_routes_to_provider,
+                llm_router,
+                custom_llm_provider,
+                frozenset(deployment.model_name for deployment in deployments),
+            ),
+        ),
+        custom_llm_provider,
+    )
+
+
+def _files_serving_model(llm_router: Router, model: str) -> _ListedFiles:
+    deployments: Final = _deployments_serving_model(llm_router, model)
+    providers: Final = frozenset(map(_resolved_deployment_provider, deployments))
+    return _ListedFiles(
+        FileListModelScope(_model_mapping_keys(deployments)), next(iter(providers)) if len(providers) == 1 else None
+    )
+
+
+async def _listed_files(
+    llm_router: Router | None,
+    user_api_key_dict: UserAPIKeyAuth,
+    custom_llm_provider: str | None,
+    target_model_names: str | None,
+    routed_model: str | None,
+) -> _ListedFiles | _UnscopedFileList:
+    if routed_model is not None and llm_router is not None:
+        return _files_serving_model(llm_router, routed_model)
+    if target_model_names:
+        target_model_name: Final = _single_target_model_name(target_model_names)
+        if isinstance(target_model_name, _UnscopedFileList):
+            return target_model_name
+        if llm_router is None:
+            return _UnscopedFileList(
+                status_code=500, detail="LLM Router not initialized. Ensure models added to proxy."
+            )
+        await get_authorized_credentials_for_model(
+            llm_router=llm_router,
+            model_id=target_model_name,
+            user_api_key_dict=user_api_key_dict,
+            operation_context="file list",
+        )
+        return _files_serving_model(llm_router, target_model_name)
+    if custom_llm_provider is None:
+        return _ListedFiles(None, None)
+    return _files_on_provider(llm_router, custom_llm_provider)
+
+
 @router.get(
     "/{provider}/v1/files",
     dependencies=[Depends(user_api_key_auth)],
@@ -1697,6 +1883,8 @@ async def list_files(
     purpose: str | None = None,
     limit: int | None = None,
     after: str | None = None,
+    after_id: str | None = None,
+    before_id: str | None = None,
 ):
     """
     Returns information about a specific file. that can be used across - Assistants API, Batch API 
@@ -1739,6 +1927,10 @@ async def list_files(
         )
 
         response: Any | None = None
+        owned_files_hook: Final = proxy_logging_obj.get_proxy_hook("managed_files")
+        list_owned_files_only: Final = litellm.require_managed_files is True and not user_api_key_has_admin_view(
+            user_api_key_dict
+        )
 
         # Check for model-based credential routing (no file_id encoding check for list)
         should_route, model_used, _, credentials = await handle_model_based_routing(
@@ -1750,7 +1942,36 @@ async def list_files(
             check_file_id_encoding=False,
         )
 
-        if should_route and credentials is not None:
+        if list_owned_files_only:
+            if after and after_id and after != after_id:
+                raise HTTPException(
+                    status_code=400,
+                    detail="Pass the list cursor as either 'after' or 'after_id', not both with different ids.",
+                )
+            if not isinstance(owned_files_hook, BaseFileEndpoints):
+                raise HTTPException(
+                    status_code=500,
+                    detail="Managed file listing is unavailable, so require_managed_files cannot scope the list to the caller.",
+                )
+            listed_files: Final = await _listed_files(
+                llm_router=llm_router,
+                user_api_key_dict=user_api_key_dict,
+                custom_llm_provider=await _requested_custom_llm_provider(request=request, provider=provider),
+                target_model_names=target_model_names,
+                routed_model=model_used if should_route else None,
+            )
+            if isinstance(listed_files, _UnscopedFileList):
+                _raise_unscoped_file_list(listed_files)
+            response = await _list_owned_managed_files(  # rebind-ok: each dispatch branch binds response
+                managed_files_obj=owned_files_hook,
+                user_api_key_dict=user_api_key_dict,
+                purpose=purpose,
+                limit=limit if limit is not None else _default_file_list_limit(listed_files.custom_llm_provider),
+                after=after or after_id,
+                before=before_id,
+                model_scope=listed_files.model_scope,
+            )
+        elif should_route and credentials is not None:
             # Use model-based routing with credentials from config
             prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True)
             response = await litellm.afile_list(
@@ -1762,12 +1983,9 @@ async def list_files(
             verbose_proxy_logger.debug("Listed files using model: %s", model_used)
 
         elif target_model_names and isinstance(target_model_names, str):
-            target_model_names_list: Final = target_model_names.split(",")
-            if len(target_model_names_list) != 1:
-                raise HTTPException(
-                    status_code=400,
-                    detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
-                )
+            target_model_name: Final = _single_target_model_name(target_model_names)
+            if isinstance(target_model_name, _UnscopedFileList):
+                _raise_unscoped_file_list(target_model_name)
             if llm_router is None:
                 raise HTTPException(
                     status_code=500,
@@ -1775,7 +1993,7 @@ async def list_files(
                 )
             credentials = await get_authorized_credentials_for_model(
                 llm_router=llm_router,
-                model_id=target_model_names_list[0],
+                model_id=target_model_name,
                 user_api_key_dict=user_api_key_dict,
                 operation_context="file list",
             )
@@ -1786,20 +2004,17 @@ async def list_files(
                 **data,
             )
         else:
-            custom_llm_provider: Final = (
-                provider
-                or get_custom_llm_provider_from_request_headers(request=request)
-                or get_custom_llm_provider_from_request_query(request=request)
-                or await get_custom_llm_provider_from_request_body(request=request)
-            )
+            custom_llm_provider: Final = await _requested_custom_llm_provider(request=request, provider=provider)
             managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files")
             if custom_llm_provider is None and isinstance(managed_files_obj, BaseFileEndpoints):
-                response = await managed_files_obj.afile_list(
-                    purpose=purpose,
-                    litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
+                response = await _list_owned_managed_files(  # rebind-ok: each dispatch branch binds response
+                    managed_files_obj=managed_files_obj,
                     user_api_key_dict=user_api_key_dict,
+                    purpose=purpose,
                     limit=limit,
                     after=after,
+                    before=None,
+                    model_scope=None,
                 )
             else:
                 resolved_custom_llm_provider: Final = custom_llm_provider or "openai"

litellm/proxy/pass_through_endpoints/managed_id_rewriter.py

@@ -242,6 +242,12 @@ def _passthrough_sentinel_model_id(provider: str) -> str:
     return f"_passthrough_{provider}"
 
 
+def passthrough_file_model_id(provider: str) -> str | None:
+    if (provider, "/v1/files") not in _LIST_ROUTE_TABLE:
+        return None
+    return _passthrough_sentinel_model_id(provider)
+
+
 # Key under which the provider marker is stored in a file row's model_mappings.
 # Its value lands in flat_model_file_ids (built from model_mappings.values()),
 # giving the file table a DB-queryable provider scope it otherwise lacks.

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -30,6 +30,7 @@ from pydantic import TypeAdapter
 from starlette.datastructures import UploadFile as StarletteUploadFile
 from starlette.routing import BaseRoute, Route
 from starlette.websockets import WebSocketState
+from typing_extensions import ReadOnly
 from websockets.asyncio.client import ClientConnection, connect
 from websockets.exceptions import (
     ConnectionClosedError,
@@ -173,8 +174,17 @@ def _is_mergeable_client_metadata(key: str, value: object) -> bool:
     return not key.startswith("user_api_key_") and (key != "model_info" or isinstance(value, Mapping))
 
 
+class _RegisteredPassThroughRoute(TypedDict):
+    endpoint_id: ReadOnly[str]
+    path: ReadOnly[str | None]
+    type: ReadOnly[str]
+    methods: ReadOnly[str | bool | list[str] | tuple[str, ...] | Mapping[str, object]]
+    auth: ReadOnly[bool]
+    passthrough_params: ReadOnly[Mapping[str, object]]
+
+
 # Global registry to track registered pass-through routes and prevent memory leaks
-_registered_pass_through_routes: Final[dict[str, dict[str, str | bool | list[str] | Mapping[str, object]]]] = {}
+_registered_pass_through_routes: Final[dict[str, _RegisteredPassThroughRoute]] = {}
 
 
 def get_response_body(response: httpx.Response) -> dict | None:
@@ -2239,7 +2249,7 @@ def create_pass_through_route(
                     status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
                     detail=f"Method {request.method} is not allowed for pass-through endpoint {path}.",
                 )
-            target_params: Final = {
+            target_params: Final[dict[str, object]] = {
                 "target": target,
                 "custom_headers": custom_headers,
                 "forward_headers": _forward_headers,
@@ -2271,7 +2281,9 @@ def create_pass_through_route(
 
             # Ensure custom_headers is a dict. Botocore returns a HeadersDict
             # for SigV4-prepared requests, which is a Mapping but not a dict.
-            headers_dict: Final = dict(param_custom_headers) if isinstance(param_custom_headers, Mapping) else {}
+            headers_dict: Final[dict[str, object]] = (
+                {**param_custom_headers} if isinstance(param_custom_headers, Mapping) else {}
+            )
 
             # Ensure query_params and custom_body are dicts or None
             final_query_params: Final = query_params_data if isinstance(query_params_data, dict) else {}
@@ -3158,9 +3170,7 @@ class SafeRouteAdder:
                 return f"{path}/{{subpath:path}}"
             return None
 
-        def registered_route_methods(
-            route_info: Mapping[str, str | bool | Sequence[str] | Mapping[str, object]],
-        ) -> tuple[tuple[str, str], ...]:
+        def registered_route_methods(route_info: _RegisteredPassThroughRoute) -> tuple[tuple[str, str], ...]:
             starlette_path: Final = registered_route_path(route_info)
             methods: Final = route_info.get("methods")
             if starlette_path is None or not isinstance(methods, (list, tuple)):
@@ -3516,7 +3526,7 @@ class InitPassThroughEndpointHelpers:
         return (comparison_route, *forwarded)
 
     @staticmethod
-    def get_registered_pass_through_route(route: str, method: str | None = None) -> dict[str, Any] | None:
+    def get_registered_pass_through_route(route: str, method: str | None = None) -> _RegisteredPassThroughRoute | None:
         """Get passthrough params for a given route and optionally filter by HTTP method"""
         comparison_route: Final = InitPassThroughEndpointHelpers._route_for_registry_lookup(route)
         for key in _registered_pass_through_routes:

litellm/proxy/proxy_server.py

@@ -666,7 +666,11 @@ from litellm.proxy.list_api.common import (
     problem_response,
     request_validation_problem,
 )
-from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request
+from litellm.proxy.litellm_pre_call_utils import (
+    SCHEDULER_PRIORITY_HEADER,
+    LiteLLMProxyRequestSetup,
+    add_litellm_data_to_request,
+)
 from litellm.proxy.logging_endpoints.callback_logs_endpoints import (
     rust_control_plane_router,
 )
@@ -18481,6 +18485,11 @@ async def async_queue_request(
             raise HTTPException(status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value})
         router: Final = llm_router
 
+        data.update(
+            router.priority_header_update(
+                model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
+            )
+        )
         queued_priority, _ = router.split_request_priority(model=data["model"], kwargs=data)
         if "stream" in data and data["stream"] is True:  # use generate_responses to stream responses
 

litellm/proxy/schema.prisma

@@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
   @@index([unified_file_id])
   @@index([flat_model_file_ids], type: Gin)
   @@index([team_id, created_at(sort: Desc)])
+  @@index([created_by, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
   @@index([window_end])
   @@index([created_at])
 }
+
+enum AccountingEventKind {
+  inbound_usage
+  outbound_usage
+  cost
+}
+
+enum AccountingDeliveryState {
+  pending
+  applied
+  parked
+}
+
+enum AccountingDeliveryTarget {
+  LiteLLM_VerificationToken
+  LiteLLM_UserTable
+  LiteLLM_TeamTable
+  LiteLLM_TeamMembership
+  LiteLLM_OrganizationTable
+  LiteLLM_OrganizationMembership
+  LiteLLM_EndUserTable
+  LiteLLM_AgentsTable
+  LiteLLM_TagTable
+  LiteLLM_ProjectTable
+  LiteLLM_ModelAccessGroupBudgetTable
+  LiteLLM_BudgetTable
+  LiteLLM_BudgetWindowSpend
+  LiteLLM_DailyUserSpend
+  LiteLLM_DailyTeamSpend
+  LiteLLM_DailyOrganizationSpend
+  LiteLLM_DailyEndUserSpend
+  LiteLLM_DailyAgentSpend
+  LiteLLM_DailyTagSpend
+  LiteLLM_DailyModelUsage
+  LiteLLM_DailyToolSpend
+  LiteLLM_AutoRouterDailySpend
+}
+
+// Durable accounting ledger. One row per usage or cost event; payload holds the
+// private event envelope versioned by schema_version. Usage rows retain request-scoped
+// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
+model LiteLLM_AccountingEvent {
+  event_id        String              @id
+  kind            AccountingEventKind
+  source_event_id String?
+  schema_version  Int
+  payload_hash    String
+  occurred_at     DateTime
+  recorded_at     DateTime            @default(now())
+  payload         Json
+
+  @@index([recorded_at])
+  @@index([kind, source_event_id])
+}
+
+// One row per (event, target table, version): pending work for the projector and,
+// once applied in the same transaction as the target write, the receipt that stops a retry.
+model LiteLLM_AccountingDelivery {
+  event_id        String
+  target          AccountingDeliveryTarget
+  version         Int
+  state           AccountingDeliveryState  @default(pending)
+  attempts        Int                      @default(0)
+  next_attempt_at DateTime                 @default(now())
+  applied_at      DateTime?
+  last_error      String?
+
+  @@id([event_id, target, version])
+  @@index([target, state, next_attempt_at])
+}

litellm/proxy/spend_tracking/key_metadata_recovery.py

@@ -6,7 +6,7 @@ from datetime import datetime, timedelta
 from types import MappingProxyType
 from typing import Final, Literal, TypeVar
 
-from pydantic import TypeAdapter
+from pydantic import ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
@@ -155,6 +155,7 @@ class _DailyUserSpendOwnerRow(LiteLLMBaseModel):
     last_owner: str | None = None
 
 
+_RAW_ROWS: Final = TypeAdapter(Sequence[dict[str, object]], config=ConfigDict(strict=True))
 _TOKEN_DIGEST_ROWS: Final = TypeAdapter(tuple[_TokenDigestRow, ...])
 _SPEND_LOG_DIGEST_ROWS: Final = TypeAdapter(tuple[_SpendLogDigestRow, ...])
 _DAILY_USER_SPEND_OWNER_ROWS: Final = TypeAdapter(tuple[_DailyUserSpendOwnerRow, ...])
@@ -194,7 +195,7 @@ async def _rows_within_the_statement_timeout(
         await transaction.execute_raw(_SPEND_LOG_STATEMENT_TIMEOUT_SQL)
         for setting in planner_settings:
             await transaction.execute_raw(setting)
-        return await transaction.query_raw(sql, *params)
+        return _RAW_ROWS.validate_python(await transaction.query_raw(sql, *params))
 
 
 async def _reverse_hash_key_metadata(

litellm/repositories/daily_activity_repository.py

@@ -19,6 +19,7 @@ from litellm.repositories.daily_activity_sql import (
     build_cache_leakage_keys_sql,
     build_entity_rollup_sql,
     build_export_sql,
+    build_guardrail_spend_sql,
     build_key_page_sql,
     build_key_search_sql,
     build_model_top_keys_sql,
@@ -36,6 +37,7 @@ from litellm.types.repositories.daily_activity import (
     ExportRow,
     ExportType,
     GroupingSetsRow,
+    GuardrailSpendRow,
     KeyMetadataRow,
     KeyPage,
     KeySpendRow,
@@ -100,6 +102,7 @@ class DailyActivityDatabase(Protocol):
 
 _GROUPING_ADAPTER: Final = TypeAdapter(tuple[GroupingSetsRow, ...])
 _ENTITY_ADAPTER: Final = TypeAdapter(tuple[EntityRollupRow, ...])
+_GUARDRAIL_SPEND_ADAPTER: Final = TypeAdapter(tuple[GuardrailSpendRow, ...])
 _KEY_SPEND_ADAPTER: Final = TypeAdapter(tuple[KeySpendRow, ...])
 _KEY_PAGE_TOTAL_ADAPTER: Final[TypeAdapter[int]] = TypeAdapter(int)
 _USER_SPEND_ADAPTER: Final = TypeAdapter(tuple[UserSpendRow, ...])
@@ -191,6 +194,9 @@ class DailyActivityRepository:
             distinct_api_keys=distinct_api_keys,
         )
 
+    async def guardrail_spend(self, scope: DailyActivityScope) -> tuple[GuardrailSpendRow, ...]:
+        return _GUARDRAIL_SPEND_ADAPTER.validate_python(await self._query(build_guardrail_spend_sql(scope)))
+
     async def search_keys(self, scope: DailyActivityScope, *, search: str, limit: int) -> tuple[str, ...]:
         if not 1 <= limit <= constants.USAGE_KEY_SEARCH_MAX:
             raise ValueError(f"limit must be between 1 and {constants.USAGE_KEY_SEARCH_MAX}")

litellm/repositories/daily_activity_sql.py

@@ -261,6 +261,25 @@ def build_entity_rollup_sql(scope: DailyActivityScope, *, api_key_limit: int) ->
     return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, api_key_limit))
 
 
+def build_guardrail_spend_sql(scope: DailyActivityScope) -> SqlQuery:
+    pg_table: Final = PRISMA_TO_PG_TABLE[scope.table]
+    where_clause, where_params = build_where_clause(scope)
+    sql: Final = f"""
+        SELECT g.date, g.guardrail_id, COALESCE(t.guardrail_name, g.guardrail_id) AS guardrail_name, g.api_key,
+               COALESCE(SUM(g.cost), 0)::float AS spend,
+               COALESCE(SUM(CASE WHEN g.cost IS NULL THEN g.units ELSE g.untracked_units END), 0)::bigint AS untracked_units
+        FROM "LiteLLM_DailyGuardrailUsageUnits" g
+        LEFT JOIN "LiteLLM_GuardrailsTable" t ON t.guardrail_id = g.guardrail_id
+        WHERE g.date >= $1 AND g.date <= $2
+          AND EXISTS (
+              SELECT 1 FROM "{pg_table}"
+              WHERE {where_clause} AND "{pg_table}".date = g.date AND "{pg_table}".api_key = g.api_key
+          )
+        GROUP BY g.date, g.guardrail_id, t.guardrail_name, g.api_key
+    """
+    return SqlQuery(sql=sql, params=where_params)
+
+
 def _key_spend_select() -> str:
     return """
             COALESCE(SUM(spend), 0)::float AS spend,

litellm/router.py

@@ -359,6 +359,7 @@ from litellm.router_utils.request_priority import (
     InvalidPriority,
     parse_default_priority,
     priority_error,
+    priority_header_fields,
     request_drops_params,
     resolve_request_priority,
     unqueueable_priority_message,
@@ -1369,7 +1370,7 @@ class Router:
             },
         ]
 
-        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
+        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": ["openai-gpt-3.5-turbo"]}])
         ```
         """
 
@@ -1595,6 +1596,10 @@ class Router:
 
         if default_fallbacks is not None or litellm.default_fallbacks is not None:
             _fallbacks = default_fallbacks or litellm.default_fallbacks
+            if isinstance(_fallbacks, str):
+                raise TypeError(
+                    f"default_fallbacks must be a list of model names, got '{_fallbacks}'. Use ['{_fallbacks}']"
+                )
             if self.fallbacks is not None:
                 self.fallbacks.append({"*": _fallbacks})
             else:
@@ -2838,7 +2843,7 @@ class Router:
         self._initialize_core_endpoints()
         self._initialize_specialized_endpoints()
 
-    def validate_fallbacks(self, fallback_param: list | None):
+    def validate_fallbacks(self, fallback_param: Sequence[Mapping[str, object]] | None):
         """
         Validate the fallbacks parameter.
         """
@@ -2851,6 +2856,13 @@ class Router:
                 raise ValueError(
                     f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys."
                 )
+            ((model_group, targets),) = fallback_dict.items()
+            if isinstance(targets, list) or check_non_standard_fallback_format(fallbacks=[fallback_dict]):
+                continue
+            raise TypeError(
+                f"Fallbacks for '{model_group}' must be a list of model names, got '{targets}'. "
+                f"Use {{'{model_group}': ['{targets}']}}"
+            )
 
     def _add_encrypted_content_affinity_check(self, enable_global_affinity: bool) -> None:
         def _move_before_deployment_affinity(
@@ -5411,14 +5423,22 @@ class Router:
         request_priority: Final = resolve_request_priority(
             requested=kwargs.get("priority"),
             default_priority=self.default_priority,
-            drops_params=lambda: request_drops_params(
-                kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
-            ),
+            drops_params=lambda: self._drops_params(model=model, kwargs=kwargs),
         )
         if isinstance(request_priority, InvalidPriority):
             raise priority_error(model=model, message=request_priority.message)
         return request_priority, {key: value for key, value in kwargs.items() if key != "priority"}
 
+    def priority_header_update(
+        self, model: str, kwargs: Mapping[str, object], header: str | None
+    ) -> Mapping[str, int | str]:
+        return priority_header_fields(header, drops_params=lambda: self._drops_params(model=model, kwargs=kwargs))
+
+    def _drops_params(self, model: str, kwargs: Mapping[str, object]) -> bool:
+        return request_drops_params(
+            kwargs, self.default_litellm_params, self._request_deployment_params(model=model, kwargs=kwargs)
+        )
+
     async def _schedule_factory(
         self,
         model: str,
@@ -11850,6 +11870,11 @@ class Router:
             indices: Final = self.model_name_to_deployment_indices.get(model_name)
             return None if indices is None else tuple(self.model_list[idx] for idx in indices)
 
+    def team_public_model_names_of(self, team_id: str) -> frozenset[str]:
+        """The public names `team_id` serves its own deployments under."""
+        with self._deployment_index_lock:
+            return frozenset(name for owner, name in self.team_model_to_deployment_indices if owner == team_id)
+
     def _team_deployments_named(self, team_id: str, public_model_name: str) -> tuple[DeploymentTypedDict, ...] | None:
         """The deployments `team_id` serves under `public_model_name`, or None when that pair is not indexed."""
         with self._deployment_index_lock:

litellm/router_utils/cooldown_handlers.py

@@ -13,7 +13,7 @@ from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm._internal_context import service_target
@@ -417,17 +417,35 @@ def _should_run_cooldown_logic(
     return True
 
 
-_FALLBACK_ENTRIES_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, list[str | Mapping[str, object]]] | str, ...])
+_FALLBACK_ENTRIES_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+_FALLBACK_ENTRY_ADAPTER: Final = TypeAdapter(Mapping[str, list[str | Mapping[str, object]] | str])
 
 
 def _fallback_target_name(target: str | Mapping[str, object]) -> str:
     return target if isinstance(target, str) else str(target.get("model", ""))
 
 
+def _fallback_target_names(
+    targets: Sequence[str | Mapping[str, object]] | str,
+) -> list[str]:  # mutable-ok: mirrors get_fallback_model_group's fallbacks shape
+    return [targets] if isinstance(targets, str) else [_fallback_target_name(target) for target in targets]
+
+
 def _fallback_chain_names(
-    entry: Mapping[str, Sequence[str | Mapping[str, object]]],
-) -> Mapping[str, list[str]]:  # mutable-ok: mirrors get_fallback_model_group's fallbacks shape
-    return {key: [_fallback_target_name(target) for target in targets] for key, targets in entry.items()}
+    entry: object,
+) -> Mapping[str, list[str]] | None:  # mutable-ok: mirrors get_fallback_model_group's fallbacks shape
+    try:
+        validated: Final = _FALLBACK_ENTRY_ADAPTER.validate_python(entry)
+    except ValidationError:
+        return None
+    return {key: _fallback_target_names(targets) for key, targets in validated.items()}
+
+
+def _fallback_entries(configured: object) -> tuple[object, ...]:
+    try:
+        return _FALLBACK_ENTRIES_ADAPTER.validate_python(configured)
+    except ValidationError:
+        return ()
 
 
 def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_group: str | None) -> bool:
@@ -437,9 +455,7 @@ def _has_fallback_group(litellm_router_instance: LitellmRouter, requested_model_
     if requested_model_group is None or not configured:
         return False
     chains: Final = tuple(
-        _fallback_chain_names(entry)
-        for entry in _FALLBACK_ENTRIES_ADAPTER.validate_python(configured)
-        if isinstance(entry, Mapping)
+        chain for chain in map(_fallback_chain_names, _fallback_entries(configured)) if chain is not None
     )
     fallback_group, _ = get_fallback_model_group(fallbacks=chains, model_group=requested_model_group)
     return bool(fallback_group)

litellm/router_utils/pre_call_checks/model_rate_limit_check.py

@@ -5,15 +5,19 @@ When enabled via router_settings.optional_pre_call_checks: ["enforce_model_rate_
 
 - tpm/rpm: combined TPM + optional RPM (legacy)
 - itpm/otpm: separate input/output tokens per minute
+- tpd/rpd: tokens and requests per UTC calendar day
 
 When a deployment sets both itpm/otpm and tpm/rpm, both are enforced. A warning
 is logged the first time such a deployment is seen.
 """
 
 import contextlib
-from typing import TYPE_CHECKING, Any, Final
+from collections.abc import Callable
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
+from pydantic import BaseModel, ConfigDict
 
 import litellm
 from litellm._internal_context import with_service_target
@@ -33,6 +37,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
     io_token_refund_failure,
 )
 from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET
+from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.router import RouterErrors
 from litellm.types.utils import StandardLoggingPayload
 from litellm.utils import get_utc_datetime
@@ -47,6 +52,62 @@ else:
 
 class RoutingArgs:
     ttl: int = 60  # 1min (RPM/TPM expire key)
+    daily_ttl: int = 86400
+
+
+DAY_FORMAT: Final = "%Y-%m-%d"
+
+
+def seconds_until_utc_midnight(now: datetime) -> int:
+    next_midnight: Final = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
+    return max(1, int((next_midnight - now).total_seconds()))
+
+
+def _first_set(*values: int | None) -> int | None:
+    return next((value for value in values if value is not None), None)
+
+
+class _DailyLimitFields(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+    tpd: int | None = None
+    rpd: int | None = None
+
+
+class _DailyLimitParams(_DailyLimitFields):
+    model: str | None = None
+
+
+class _DailyLimitModelInfo(_DailyLimitFields):
+    id: str | None = None
+
+
+class _DailyLimitDeployment(_DailyLimitFields):
+    model_name: str = ""
+    litellm_params: _DailyLimitParams = _DailyLimitParams()
+    model_info: _DailyLimitModelInfo = _DailyLimitModelInfo()
+
+    def limits(self) -> tuple[int | None, int | None]:
+        return (
+            _first_set(self.tpd, self.litellm_params.tpd, self.model_info.tpd),
+            _first_set(self.rpd, self.litellm_params.rpd, self.model_info.rpd),
+        )
+
+    def cache_keys(self, current_day: str) -> tuple[str, str]:
+        prefix: Final = f"{self.model_info.id}:{self.litellm_params.model}"
+        return f"{prefix}:tpd:{current_day}", f"{prefix}:rpd:{current_day}"
+
+    def limit_error(self, kind: Literal["tpd", "rpd"], limit: int, usage: int, now: datetime) -> litellm.RateLimitError:
+        return litellm.RateLimitError(
+            message=f"Model rate limit exceeded. {kind.upper()} limit={limit}, current usage={usage}",
+            llm_provider="",
+            model=self.litellm_params.model,
+            response=httpx.Response(
+                status_code=429,
+                content=f"{RouterErrors.user_defined_ratelimit_error.value} {kind} limit={limit}. current usage={usage}. id={self.model_info.id}, model_group={self.model_name}",
+                headers={"retry-after": str(seconds_until_utc_midnight(now))},
+                request=httpx.Request(method="model_rate_limit_check", url="https://github.com/BerriAI/litellm"),
+            ),
+        )
 
 
 class ModelRateLimitingCheck(CustomLogger):
@@ -60,12 +121,60 @@ class ModelRateLimitingCheck(CustomLogger):
     this check actively enforces those limits across ALL routing strategies.
     """
 
-    def __init__(self, dual_cache: DualCache):
+    def __init__(self, dual_cache: DualCache, clock: Callable[[], datetime] = get_utc_datetime):
         self.dual_cache = dual_cache
+        self.clock = clock
         # model_ids already warned about itpm/otpm + tpm/rpm on the same deployment,
         # so the warning is logged once per deployment rather than per request.
         self._io_token_conflict_warned_ids: set[str] = set()
 
+    def _increment(self, key: str, value: int, ttl: int) -> int:
+        return self.dual_cache.increment_cache(key=key, value=value, ttl=ttl)
+
+    async def _async_increment(
+        self, key: str, value: int, ttl: int, parent_otel_span: Span | None = None
+    ) -> int | None:
+        count: Final = await self.dual_cache.async_increment_cache(
+            key=key, value=value, ttl=ttl, parent_otel_span=parent_otel_span
+        )
+        return None if count is None else int(count)
+
+    @staticmethod
+    def _request_count_operations(
+        rpm_key: str | None, rpd_key: str | None, increment_value: int
+    ) -> tuple[RedisPipelineIncrementOperation, ...]:
+        return tuple(
+            RedisPipelineIncrementOperation(key=key, increment_value=increment_value, ttl=ttl)
+            for key, ttl in ((rpm_key, RoutingArgs.ttl), (rpd_key, RoutingArgs.daily_ttl))
+            if key is not None
+        )
+
+    async def _async_refund_request_counts(
+        self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None
+    ) -> None:
+        operations: Final = self._request_count_operations(rpm_key, rpd_key, -1)
+        if operations:
+            await self.dual_cache.async_increment_cache_pipeline(
+                increment_list=list(operations), parent_otel_span=parent_otel_span
+            )
+
+    async def _async_increment_request_counts(
+        self, rpm_key: str | None, rpd_key: str | None, parent_otel_span: Span | None
+    ) -> tuple[int | None, int | None]:
+        operations: Final = self._request_count_operations(rpm_key, rpd_key, 1)
+        if not operations:
+            return None, None
+        counts: Final = await self.dual_cache.async_increment_cache_pipeline(
+            increment_list=list(operations), parent_otel_span=parent_otel_span
+        )
+        if counts is None or len(counts) != len(operations):
+            return None, None
+        count_by_key: Final = {op["key"]: int(count) for op, count in zip(operations, counts)}
+        return (
+            None if rpm_key is None else count_by_key.get(rpm_key),
+            None if rpd_key is None else count_by_key.get(rpd_key),
+        )
+
     def _warn_io_token_and_tpm_rpm_coexist_once(self, deployment: dict) -> None:
         tpm_limit, rpm_limit = self._get_deployment_limits(deployment)
         if tpm_limit is None and rpm_limit is None:
@@ -129,6 +238,21 @@ class ModelRateLimitingCheck(CustomLogger):
 
         return tpm, rpm
 
+    def _check_daily_limits(self, deployment: _DailyLimitDeployment, now: datetime) -> None:
+        tpd_limit, rpd_limit = deployment.limits()
+        if tpd_limit is None and rpd_limit is None:
+            return
+        tpd_key, rpd_key = deployment.cache_keys(now.strftime(DAY_FORMAT))
+        if tpd_limit is not None:
+            current_tpd: Final = self._get_current_tpm(tpd_key, tpd_limit)
+            if current_tpd is not None and current_tpd >= tpd_limit:
+                raise deployment.limit_error("tpd", tpd_limit, current_tpd, now)
+        if rpd_limit is not None:
+            current_rpd: Final = self._increment(rpd_key, 1, RoutingArgs.daily_ttl)
+            if current_rpd > rpd_limit:
+                self._increment(rpd_key, -1, RoutingArgs.daily_ttl)
+                raise deployment.limit_error("rpd", rpd_limit, current_rpd, now)
+
     def _get_cache_keys(self, deployment: dict, current_minute: str) -> tuple[str, str]:
         """Get the cache keys for TPM and RPM tracking."""
         model_id: Final = deployment.get("model_info", {}).get("id")
@@ -178,13 +302,10 @@ class ModelRateLimitingCheck(CustomLogger):
                 )
                 io_reservation_made = True
 
+            dt: Final = self.clock()
+            daily: Final = _DailyLimitDeployment.model_validate(deployment)
             tpm_limit, rpm_limit = self._get_deployment_limits(deployment)
 
-            # If no limits are set, allow the request
-            if tpm_limit is None and rpm_limit is None:
-                return deployment
-
-            dt: Final = get_utc_datetime()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute)
 
@@ -213,8 +334,9 @@ class ModelRateLimitingCheck(CustomLogger):
 
             # Check RPM limit (atomic increment-first to avoid race conditions)
             if rpm_limit is not None:
-                current_rpm: Final = self.dual_cache.increment_cache(key=rpm_key, value=1, ttl=RoutingArgs.ttl)
+                current_rpm: Final = self._increment(rpm_key, 1, RoutingArgs.ttl)
                 if current_rpm is not None and current_rpm > rpm_limit:
+                    self._increment(rpm_key, -1, RoutingArgs.ttl)
                     raise litellm.RateLimitError(
                         message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}",
                         llm_provider="",
@@ -230,6 +352,12 @@ class ModelRateLimitingCheck(CustomLogger):
                         ),
                     )
 
+            try:
+                self._check_daily_limits(daily, dt)
+            except litellm.RateLimitError:
+                if rpm_limit is not None:
+                    self._increment(rpm_key, -1, RoutingArgs.ttl)
+                raise
             return deployment
 
         except litellm.RateLimitError:
@@ -259,15 +387,14 @@ class ModelRateLimitingCheck(CustomLogger):
                 )
                 io_reservation_made = True
 
+            dt: Final = self.clock()
+            daily: Final = _DailyLimitDeployment.model_validate(deployment)
+            tpd_limit, rpd_limit = daily.limits()
             tpm_limit, rpm_limit = self._get_deployment_limits(deployment)
 
-            # If no limits are set, allow the request
-            if tpm_limit is None and rpm_limit is None:
-                return deployment
-
-            dt: Final = get_utc_datetime()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key, rpm_key = self._get_cache_keys(deployment, current_minute)
+            tpd_key, rpd_key = daily.cache_keys(dt.strftime(DAY_FORMAT))
 
             model_id: Final = deployment.get("model_info", {}).get("id")
             model_name: Final = deployment.get("litellm_params", {}).get("model")
@@ -293,30 +420,37 @@ class ModelRateLimitingCheck(CustomLogger):
                         num_retries=0,  # Don't retry - return 429 immediately
                     )
 
-            # Check RPM limit (atomic increment-first to avoid race conditions)
-            if rpm_limit is not None:
-                current_rpm: Final = await self.dual_cache.async_increment_cache(
-                    key=rpm_key,
-                    value=1,
-                    ttl=RoutingArgs.ttl,
-                    parent_otel_span=parent_otel_span,
-                )
-                if current_rpm is not None and current_rpm > rpm_limit:
-                    raise litellm.RateLimitError(
-                        message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}",
-                        llm_provider="",
-                        model=model_name,
-                        response=httpx.Response(
-                            status_code=429,
-                            content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}",
-                            headers={"retry-after": str(60)},
-                            request=httpx.Request(
-                                method="model_rate_limit_check",
-                                url="https://github.com/BerriAI/litellm",
-                            ),
+            if tpd_limit is not None:
+                current_tpd: Final = await self._async_get_current_tpm(tpd_key, tpd_limit, parent_otel_span)
+                if current_tpd is not None and current_tpd >= tpd_limit:
+                    raise daily.limit_error("tpd", tpd_limit, current_tpd, dt)
+
+            charged_rpm_key: Final = rpm_key if rpm_limit is not None else None
+            charged_rpd_key: Final = rpd_key if rpd_limit is not None else None
+            current_rpm, current_rpd = await self._async_increment_request_counts(
+                charged_rpm_key, charged_rpd_key, parent_otel_span
+            )
+            if rpm_limit is not None and current_rpm is not None and current_rpm > rpm_limit:
+                await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span)
+                raise litellm.RateLimitError(
+                    message=f"Model rate limit exceeded. RPM limit={rpm_limit}, current usage={current_rpm}",
+                    llm_provider="",
+                    model=model_name,
+                    response=httpx.Response(
+                        status_code=429,
+                        content=f"{RouterErrors.user_defined_ratelimit_error.value} rpm limit={rpm_limit}. current usage={current_rpm}. id={model_id}, model_group={model_group}",
+                        headers={"retry-after": str(60)},
+                        request=httpx.Request(
+                            method="model_rate_limit_check",
+                            url="https://github.com/BerriAI/litellm",
                         ),
-                        num_retries=0,  # Don't retry - return 429 immediately
-                    )
+                    ),
+                    num_retries=0,  # Don't retry - return 429 immediately
+                )
+
+            if rpd_limit is not None and current_rpd is not None and current_rpd > rpd_limit:
+                await self._async_refund_request_counts(charged_rpm_key, charged_rpd_key, parent_otel_span)
+                raise daily.limit_error("rpd", rpd_limit, current_rpd, dt)
 
             return deployment
 
@@ -373,17 +507,16 @@ class ModelRateLimitingCheck(CustomLogger):
             if not model or not total_tokens:
                 return
 
-            dt: Final = get_utc_datetime()
+            dt: Final = self.clock()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
+            tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
 
             verbose_router_logger.debug("[TPM TRACKING] Incrementing %s by %s", tpm_key, total_tokens)
 
-            await self.dual_cache.async_increment_cache(
-                key=tpm_key,
-                value=total_tokens,
-                ttl=RoutingArgs.ttl,
-            )
+            tokens: Final = int(total_tokens)
+            await self._async_increment(tpm_key, tokens, RoutingArgs.ttl)
+            await self._async_increment(tpd_key, tokens, RoutingArgs.daily_ttl)
 
         except Exception as e:
             verbose_router_logger.debug("Error in ModelRateLimitingCheck.async_log_success_event: %s", e)
@@ -434,15 +567,14 @@ class ModelRateLimitingCheck(CustomLogger):
             if not model or not total_tokens:
                 return
 
-            dt: Final = get_utc_datetime()
+            dt: Final = self.clock()
             current_minute: Final = dt.strftime("%H-%M")
             tpm_key: Final = f"{model_id}:{model}:tpm:{current_minute}"
+            tpd_key: Final = f"{model_id}:{model}:tpd:{dt.strftime(DAY_FORMAT)}"
 
-            self.dual_cache.increment_cache(
-                key=tpm_key,
-                value=total_tokens,
-                ttl=RoutingArgs.ttl,
-            )
+            tokens: Final = int(total_tokens)
+            self._increment(tpm_key, tokens, RoutingArgs.ttl)
+            self._increment(tpd_key, tokens, RoutingArgs.daily_ttl)
 
         except Exception as e:
             verbose_router_logger.debug("Error in ModelRateLimitingCheck.log_success_event: %s", e)

litellm/router_utils/request_priority.py

@@ -47,6 +47,15 @@ def resolve_request_priority(
     return priority
 
 
+def priority_header_fields(header: str | None, drops_params: Callable[[], bool]) -> Mapping[str, int | str]:
+    if not header:
+        return {}
+    priority: Final = _as_priority(header)
+    if isinstance(priority, InvalidPriority):
+        return {} if drops_params() else {"priority": header}
+    return {"priority": priority}
+
+
 def parse_default_priority(value: object) -> int | None | InvalidPriority:
     if value is None:
         return None

litellm/secret_managers/aws_secret_manager_v2.py

@@ -21,6 +21,8 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -41,6 +43,15 @@ if TYPE_CHECKING:
     from botocore.awsrequest import HTTPHeaders
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _SecretStringResponse(TypedDict):
+    SecretString: ReadOnly[str]
+
+
+_SECRET_STRING_RESPONSE: Final = TypeAdapter(_SecretStringResponse)
+_AWS_SECRET_MANAGER_RESPONSE: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
+
 class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
     def __init__(
         self,
@@ -154,7 +165,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
         try:
             response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
             response.raise_for_status()
-            return response.json()["SecretString"]
+            return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"]
         except httpx.TimeoutException:
             raise ValueError("Timeout error occurred")
         except Exception as e:
@@ -204,7 +215,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
 
         try:
             response: Final = sync_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
-            return response.json()["SecretString"]
+            return _SECRET_STRING_RESPONSE.validate_python(response.json())["SecretString"]
         except httpx.TimeoutException:
             raise ValueError("Timeout error occurred")
         except httpx.HTTPStatusError as e:
@@ -424,7 +435,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
         try:
             response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
             response.raise_for_status()
-            return response.json()
+            return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
         except httpx.HTTPStatusError as err:
             raise ValueError(f"HTTP error occurred: {err.response.text}")
         except httpx.TimeoutException:
@@ -481,7 +492,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
         try:
             response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
             response.raise_for_status()
-            return response.json()
+            return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
         except httpx.HTTPStatusError as err:
             raise ValueError(f"HTTP error occurred: {err.response.text}")
         except httpx.TimeoutException:
@@ -533,7 +544,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
         try:
             response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
             response.raise_for_status()
-            return response.json()
+            return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
         except httpx.HTTPStatusError as err:
             raise ValueError(f"HTTP error occurred: {err.response.text}")
         except httpx.TimeoutException:
@@ -613,7 +624,7 @@ class AWSSecretsManagerV2(BaseAWSLLM, BaseSecretManager):
         try:
             response: Final = await async_client.post(url=endpoint_url, headers=headers, data=body.decode("utf-8"))
             response.raise_for_status()
-            return response.json()
+            return _AWS_SECRET_MANAGER_RESPONSE.validate_python(response.json())
         except httpx.HTTPStatusError as err:
             raise ValueError(f"HTTP error occurred: {err.response.text}")
         except httpx.TimeoutException:

litellm/types/integrations/rag/bedrock_knowledgebase.py

@@ -1,4 +1,4 @@
-from typing import Any, Literal
+from typing import Literal
 
 from typing_extensions import ReadOnly, TypedDict
 
@@ -40,7 +40,7 @@ class BedrockKBRetrievalResult(TypedDict, total=False):
     content: BedrockKBContent | None
     location: BedrockKBLocation | None
     score: float | None
-    metadata: dict[str, Any] | None
+    metadata: dict[str, object] | None
 
 
 class BedrockKBResponse(TypedDict, total=False):
@@ -87,21 +87,21 @@ class BedrockKBMetadataConfiguration(TypedDict, total=False):
 class BedrockKBModelConfiguration(TypedDict, total=False):
     """Model configuration for reranking."""
 
-    additionalModelRequestFields: dict[str, Any] | None
+    additionalModelRequestFields: dict[str, object] | None
     modelArn: str | None
 
 
 class BedrockKBRerankingConfiguration(TypedDict, total=False):
     """Configuration for reranking in vector search."""
 
-    bedrockRerankingConfiguration: dict[str, Any] | None  # This could be further typed if needed
+    bedrockRerankingConfiguration: dict[str, object] | None  # This could be further typed if needed
     type: str | None
 
 
 class BedrockKBVectorSearchConfiguration(TypedDict, total=False):
     """Configuration for vector search."""
 
-    filter: dict[str, Any] | None
+    filter: dict[str, object] | None
     implicitFilterConfiguration: BedrockKBImplicitFilterConfiguration | None
     numberOfResults: int | None
     overrideSearchType: str | None

litellm/types/litellm_params.py

@@ -189,6 +189,8 @@ class DeploymentOptions:
     tpm: int | None = None
     itpm: int | None = None
     otpm: int | None = None
+    rpd: int | None = None
+    tpd: int | None = None
     kubernetes_pod_discovery: bool | None = None
     default_api_key_rpm_limit: int | None = None
     default_api_key_tpm_limit: int | None = None

litellm/types/llms/bedrock_invoke_agents.py

@@ -4,7 +4,7 @@ Type definitions for AWS Bedrock Invoke Agent API responses.
 https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html
 """
 
-from typing import Any, Final
+from typing import Final
 
 from typing_extensions import TypedDict
 
@@ -39,7 +39,7 @@ class InvokeAgentModelInvocationInput(TypedDict, total=False):
     """Model invocation input details."""
 
     foundationModel: str | None
-    inferenceConfiguration: dict[str, Any] | None
+    inferenceConfiguration: dict[str, object] | None
     text: str | None
     traceId: str | None
     type: str | None
@@ -49,9 +49,9 @@ class InvokeAgentModelInvocationOutput(TypedDict, total=False):
     """Model invocation output details."""
 
     metadata: InvokeAgentMetadata | None
-    parsedResponse: dict[str, Any] | None
-    rawResponse: dict[str, Any] | None
-    reasoningContent: dict[str, Any] | None
+    parsedResponse: dict[str, object] | None
+    rawResponse: dict[str, object] | None
+    reasoningContent: dict[str, object] | None
     traceId: str | None
 
 

litellm/types/llms/gemini.py

@@ -1,5 +1,6 @@
+from collections.abc import Mapping
 from enum import Enum
-from typing import Any, Literal
+from typing import Literal
 
 from typing_extensions import Required, TypedDict
 
@@ -235,7 +236,7 @@ class GeminiVideoGenerationInstance(TypedDict, total=False):
     """Instance data for Gemini video generation request"""
 
     prompt: Required[str]
-    image: dict[str, Any]
+    image: Mapping[str, object]
 
 
 class GeminiVideoGenerationParameters(LiteLLMBaseModel):
@@ -263,7 +264,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel):
     negativePrompt: str | None = None
     """Text describing what not to include in the video."""
 
-    lastFrame: Any | None = None
+    lastFrame: object | None = None
     """
     The final image for interpolation video to transition.
     Should be used with the 'image' parameter.
@@ -275,7 +276,7 @@ class GeminiVideoGenerationParameters(LiteLLMBaseModel):
     Only supported in Veo 3.1 (list of VideoGenerationReferenceImage objects).
     """
 
-    video: Any | None = None
+    video: object | None = None
     """
     Video to be used for video extension (Video object).
     Only supported in Veo 3.1 & Veo 3 Fast.
@@ -354,5 +355,5 @@ class GeminiLongRunningOperationResponse(LiteLLMBaseModel):
     response: GeminiOperationResponse | None = None
     """Response object when operation is complete"""
 
-    error: dict[str, Any] | None = None
+    error: dict[str, object] | None = None
     """Error details if operation failed"""

litellm/types/llms/langgraph.py

@@ -4,7 +4,8 @@ Type definitions for LangGraph API.
 LangGraph provides a streaming and non-streaming API for running agents.
 """
 
-from typing import Any, Literal
+from collections.abc import Mapping
+from typing import Literal
 
 from typing_extensions import TypedDict
 
@@ -29,8 +30,8 @@ class LangGraphRequest(TypedDict, total=False):
     assistant_id: str
     input: LangGraphInput
     stream_mode: str | None
-    config: dict[str, Any] | None
-    metadata: dict[str, Any] | None
+    config: Mapping[str, object] | None
+    metadata: Mapping[str, object] | None
 
 
 # Response Types - Streaming
@@ -38,7 +39,7 @@ class LangGraphStreamEvent(TypedDict, total=False):
     """Single event in a LangGraph stream response."""
 
     event: str
-    data: Any
+    data: object
 
 
 # Response Types - Non-streaming
@@ -55,7 +56,7 @@ class LangGraphResponse(TypedDict, total=False):
     """Non-streaming response structure from LangGraph."""
 
     messages: list[LangGraphResponseMessage]
-    values: dict[str, Any]
+    values: dict[str, object]
 
 
 # Parsed response for internal use

litellm/types/mcp_server/tool_registry.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable
-from typing import Any, ClassVar
+from typing import ClassVar
 
 from pydantic import ConfigDict, Field
 
@@ -10,7 +10,7 @@ class MCPTool(LiteLLMBaseModel):
     model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)
     name: str
     description: str
-    input_schema: dict[str, Any]
+    input_schema: dict[str, object]
     handler: Callable
     server_id: str | None = Field(default=None, frozen=True)
 
@@ -18,18 +18,18 @@ class MCPTool(LiteLLMBaseModel):
 class ToolSchema(LiteLLMBaseModel):
     name: str
     description: str
-    inputSchema: dict[str, Any]
+    inputSchema: dict[str, object]
 
 
 class ListToolsResponse(LiteLLMBaseModel):
     tools: list[ToolSchema]
     nextCursor: str | None = None
-    _meta: dict[str, Any] | None = None
+    _meta: dict[str, object] | None = None
 
 
 class CallToolRequest(LiteLLMBaseModel):
     method: str = "tools/call"
-    params: dict[str, Any]
+    params: dict[str, object]
 
 
 class ContentItem(LiteLLMBaseModel):

litellm/types/proxy/guardrails/guardrail_hooks/enkryptai.py

@@ -1,5 +1,3 @@
-from typing import Any
-
 from pydantic import Field
 from typing_extensions import TypedDict
 
@@ -20,7 +18,7 @@ class EnkryptAIPolicyViolationDetail(TypedDict, total=False):
 class EnkryptAIPIIDetail(TypedDict, total=False):
     """Details for PII detection."""
 
-    pii: dict[str, Any]
+    pii: dict[str, object]
 
 
 class EnkryptAIToxicityDetail(TypedDict, total=False):
@@ -85,8 +83,8 @@ class EnkryptAIResponseDetails(TypedDict, total=False):
     toxicity: EnkryptAIToxicityDetail | None
     keyword_detected: EnkryptAIKeywordDetail | None
     bias: EnkryptAIBiasDetail | None
-    prompt_injection: dict[str, Any] | None
-    jailbreak: dict[str, Any] | None
+    prompt_injection: dict[str, object] | None
+    jailbreak: dict[str, object] | None
 
 
 class EnkryptAIResponse(TypedDict, total=False):
@@ -107,7 +105,7 @@ class EnkryptAIProcessedResult(TypedDict):
         | EnkryptAIToxicityDetail
         | EnkryptAIKeywordDetail
         | EnkryptAIBiasDetail
-        | dict[str, Any],
+        | dict[str, object],
     ]
 
 

litellm/types/proxy/management_endpoints/common_daily_activity.py

@@ -65,6 +65,34 @@ class MetricWithMetadata(MetricBase):
     api_key_breakdown: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict)  # api_key -> {metrics, metadata}
 
 
+class GuardrailSpendMetrics(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    spend: float = 0.0
+    untracked_units: int = 0
+
+
+class GuardrailKeyMetric(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    metrics: GuardrailSpendMetrics
+    metadata: KeyMetadata
+
+
+class GuardrailMetadata(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    guardrail_name: str
+
+
+class GuardrailMetric(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    metrics: GuardrailSpendMetrics
+    metadata: GuardrailMetadata
+    api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(default_factory=dict)
+
+
 class BreakdownMetrics(LiteLLMBaseModel):
     """Breakdown of spend by different dimensions"""
 
@@ -75,6 +103,7 @@ class BreakdownMetrics(LiteLLMBaseModel):
     endpoints: dict[str, MetricWithMetadata] = Field(default_factory=dict)  # endpoint -> {metrics, metadata}
     api_keys: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict)  # api_key -> {metrics, metadata}
     entities: dict[str, MetricWithMetadata] = Field(default_factory=dict)  # entity -> {metrics, metadata}
+    guardrails: Mapping[str, GuardrailMetric] = Field(default_factory=dict)  # guardrail_id -> spend
 
 
 class DailySpendData(LiteLLMBaseModel):

litellm/types/repositories/daily_activity.py

@@ -168,6 +168,16 @@ class EntityRollupRow(RollupMetricsRow):
     distinct_api_keys: int | None
 
 
+@dataclass(frozen=True, slots=True)
+class GuardrailSpendRow:
+    date: str
+    guardrail_id: str
+    guardrail_name: str
+    api_key: str
+    spend: float
+    untracked_units: int
+
+
 @dataclass(frozen=True, slots=True)
 class AggregatedRows:
     grouping_rows: tuple[GroupingSetsRow, ...]

litellm/types/router.py

@@ -533,6 +533,8 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     rpm: int | None = None
     itpm: int | None = None
     otpm: int | None = None
+    tpd: int | None = None
+    rpd: int | None = None
     timeout: float | str | httpx.Timeout | SDKTimeout | None = None  # if str, pass in as os.environ/
     # read timeout of a live stream, a faked stream takes the longer of this and timeout. If str, pass in as os.environ/
     stream_timeout: float | str | None = None
@@ -738,6 +740,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
     rpm: int | None
     itpm: int | None
     otpm: int | None
+    tpd: ReadOnly[int | None]
+    rpd: ReadOnly[int | None]
     order: int | None
     weight: int | None
     max_parallel_requests: int | None

litellm/utils.py

@@ -10418,6 +10418,13 @@ class ProviderConfigManager:
             return get_modelscope_image_generation_config(model)
         elif LlmProviders.EDENAI == provider:
             return litellm.EdenAIImageGenerationConfig()
+        elif LlmProviders.TENCENT == provider:
+            from litellm.llms.tencent.image_generation.transformation import (
+                TencentImageGenerationConfig,
+                is_tokenhub_image_model,
+            )
+
+            return TencentImageGenerationConfig() if is_tokenhub_image_model(model) else None
         return None
 
     @staticmethod
@@ -10606,6 +10613,15 @@ class ProviderConfigManager:
             )
 
             return get_openrouter_image_edit_config(model)
+        elif LlmProviders.TENCENT == provider:
+            from litellm.llms.tencent.image_edit.transformation import (
+                TencentImageEditConfig,
+            )
+            from litellm.llms.tencent.image_generation.transformation import (
+                is_tokenhub_image_model,
+            )
+
+            return TencentImageEditConfig() if is_tokenhub_image_model(model) else None
         return None
 
     @staticmethod

model_prices_and_context_window.json

@@ -21589,7 +21589,13 @@
         ],
         "supports_vision": true,
         "supports_audio_input": true,
-        "supports_video_input": true
+        "supports_video_input": true,
+        "input_cost_per_token_priority": 5.6252e-07,
+        "output_cost_per_token_priority": 3.37498e-06,
+        "cache_read_input_token_cost_priority": 5.628e-08,
+        "cache_creation_input_token_cost_priority": 5.6252e-07,
+        "input_cost_per_audio_token_priority": 1.12497e-06,
+        "cache_read_input_audio_token_cost_priority": 1.1249e-07
     },
     "databricks/databricks-gemini-3-1-flash-image": {
         "cache_creation_input_token_cost": 6.2503e-07,
@@ -21676,7 +21682,14 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_video_input": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.50002e-06,
+        "output_cost_per_token_priority": 2.699998e-05,
+        "cache_read_input_token_cost_priority": 4.5003e-07,
+        "cache_creation_input_token_cost_priority": 4.50002e-06,
+        "input_cost_per_token_above_200k_tokens_priority": 8.99997e-06,
+        "output_cost_per_token_above_200k_tokens_priority": 4.049997e-05,
+        "cache_read_input_token_cost_above_200k_tokens_priority": 8.9999e-07
     },
     "databricks/databricks-gemini-3-flash": {
         "cache_creation_input_token_cost": 6.2503e-07,
@@ -21710,7 +21723,13 @@
         ],
         "supports_vision": true,
         "supports_audio_input": true,
-        "supports_video_input": true
+        "supports_video_input": true,
+        "input_cost_per_token_priority": 1.12497e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.1249e-07,
+        "cache_creation_input_token_cost_priority": 1.12497e-06,
+        "input_cost_per_audio_token_priority": 2.25001e-06,
+        "cache_read_input_audio_token_cost_priority": 2.2498e-07
     },
     "databricks/databricks-gemini-3-pro": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -21757,7 +21776,11 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-7-flash": {
         "cache_creation_input_token_cost": 7.4998e-07,
@@ -21785,10 +21808,14 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-6-flash": {
-        "cache_creation_input_token_cost": 1.87502e-06,
+        "cache_creation_input_token_cost": 7.4998e-07,
         "cache_read_input_token_cost": 7.497e-08,
         "input_cost_per_token": 7.4998e-07,
         "input_dbu_cost_per_token": 1.0714e-05,
@@ -21816,7 +21843,11 @@
         "supports_reasoning": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.35002e-06,
+        "output_cost_per_token_priority": 6.75003e-06,
+        "cache_read_input_token_cost_priority": 1.3503e-07,
+        "cache_creation_input_token_cost_priority": 1.35002e-06
     },
     "databricks/databricks-gemini-3-5-flash": {
         "cache_creation_input_token_cost": 1.87502e-06,
@@ -21846,7 +21877,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 3.37498e-06,
+        "output_cost_per_token_priority": 2.025002e-05,
+        "cache_read_input_token_cost_priority": 3.3747e-07,
+        "cache_creation_input_token_cost_priority": 3.37498e-06
     },
     "databricks/databricks-gemini-3-5-flash-lite": {
         "cache_creation_input_token_cost": 3.7499e-07,
@@ -21878,7 +21913,11 @@
         "supports_response_schema": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 6.7501e-07,
+        "output_cost_per_token_priority": 5.62499e-06,
+        "cache_read_input_token_cost_priority": 6.748e-08,
+        "cache_creation_input_token_cost_priority": 6.7501e-07
     },
     "databricks/databricks-gemma-3-12b": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -21921,7 +21960,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": false,
+        "input_cost_per_token_priority": 2.45e-06,
+        "output_cost_per_token_priority": 7.7e-06,
+        "cache_read_input_token_cost_priority": 4.55e-07,
+        "cache_creation_input_token_cost_priority": 2.45e-06
     },
     "databricks/databricks-glm-5-3": {
         "cache_creation_input_token_cost": 1.4e-06,
@@ -21951,7 +21994,11 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": false,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 2.45e-06,
+        "output_cost_per_token_priority": 7.7e-06,
+        "cache_read_input_token_cost_priority": 4.55e-07,
+        "cache_creation_input_token_cost_priority": 2.45e-06
     },
     "databricks/databricks-glm-5-3-flash": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -21980,7 +22027,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 2.625e-07,
+        "output_cost_per_token_priority": 8.75e-07,
+        "cache_read_input_token_cost_priority": 5.25e-08,
+        "cache_creation_input_token_cost_priority": 2.625e-07
     },
     "databricks/databricks-gpt-5": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -21999,7 +22050,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 2.49998e-06,
+        "output_cost_per_token_priority": 1.999998e-05,
+        "cache_read_input_token_cost_priority": 2.4997e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06
     },
     "databricks/databricks-gpt-5-1": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -22018,7 +22073,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 2.49998e-06,
+        "output_cost_per_token_priority": 1.999998e-05,
+        "cache_read_input_token_cost_priority": 2.4997e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06
     },
     "databricks/databricks-gpt-5-2": {
         "cache_creation_input_token_cost": 1.75e-06,
@@ -22037,7 +22096,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 3.5e-06,
+        "output_cost_per_token_priority": 2.8e-05,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "cache_creation_input_token_cost_priority": 3.5e-06
     },
     "databricks/databricks-gpt-5-3-codex": {
         "cache_creation_input_token_cost": 1.75e-06,
@@ -22056,7 +22119,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 3.5e-06,
+        "output_cost_per_token_priority": 2.8e-05,
+        "cache_read_input_token_cost_priority": 3.5e-07,
+        "cache_creation_input_token_cost_priority": 3.5e-06
     },
     "databricks/databricks-gpt-5-4": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -22087,7 +22154,14 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.00003e-06,
+        "output_cost_per_token_priority": 2.999997e-05,
+        "cache_read_input_token_cost_priority": 5.0001e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 5.00003e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.999997e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 5.0001e-07
     },
     "databricks/databricks-gpt-5-4-mini": {
         "cache_creation_input_token_cost": 7.4998e-07,
@@ -22115,7 +22189,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.50003e-06,
+        "output_cost_per_token_priority": 8.99997e-06,
+        "cache_read_input_token_cost_priority": 1.5001e-07,
+        "cache_creation_input_token_cost_priority": 1.50003e-06
     },
     "databricks/databricks-gpt-5-4-nano": {
         "cache_creation_input_token_cost": 1.9999e-07,
@@ -22175,7 +22253,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 8.00002e-06,
+        "output_cost_per_token_priority": 4.000003e-05,
+        "cache_read_input_token_cost_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_priority": 9.99999e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 1.599997e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 6.000001e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.59999e-06,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 1.999998e-05
     },
     "databricks/databricks-gpt-5-6-terra": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -22207,7 +22293,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 2.399999e-05,
+        "cache_read_input_token_cost_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 3.600002e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-5-6-luna": {
         "cache_creation_input_token_cost": 2.4997e-07,
@@ -22239,7 +22333,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 3.9998e-07,
+        "output_cost_per_token_priority": 2.40002e-06,
+        "cache_read_input_token_cost_priority": 3.997e-08,
+        "cache_creation_input_token_cost_priority": 5.0001e-07,
+        "input_cost_per_token_above_272k_tokens_priority": 8.0003e-07,
+        "output_cost_per_token_above_272k_tokens_priority": 3.60003e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.001e-08,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 1.00002e-06
     },
     "databricks/databricks-gpt-5-5": {
         "cache_creation_input_token_cost": 5.00003e-06,
@@ -22270,7 +22372,14 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.249997e-05,
+        "output_cost_per_token_priority": 7.50001e-05,
+        "cache_read_input_token_cost_priority": 1.24999e-06,
+        "cache_creation_input_token_cost_priority": 1.249997e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 1.249997e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 7.50001e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.24999e-06
     },
     "databricks/databricks-gpt-5-5-pro": {
         "cache_creation_input_token_cost": 2.999997e-05,
@@ -22319,7 +22428,11 @@
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_function_calling": true,
         "supports_prompt_caching": true,
-        "supports_tool_choice": true
+        "supports_tool_choice": true,
+        "input_cost_per_token_priority": 4.5003e-07,
+        "output_cost_per_token_priority": 3.60003e-06,
+        "cache_read_input_token_cost_priority": 4.501e-08,
+        "cache_creation_input_token_cost_priority": 4.5003e-07
     },
     "databricks/databricks-gpt-5-nano": {
         "cache_creation_input_token_cost": 4.998e-08,
@@ -22469,7 +22582,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.25e-06,
+        "output_cost_per_token_priority": 2.625e-05,
+        "cache_read_input_token_cost_priority": 5.25e-07,
+        "cache_creation_input_token_cost_priority": 5.25e-06
     },
     "databricks/system.ai.kimi-k3": {
         "litellm_provider": "databricks",
@@ -22551,7 +22668,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": false,
-        "thinking_always_on": true
+        "thinking_always_on": true,
+        "input_cost_per_token_priority": 3.85e-07,
+        "output_cost_per_token_priority": 3.85e-06,
+        "cache_read_input_token_cost_priority": 3.85e-07,
+        "cache_creation_input_token_cost_priority": 3.85e-07
     },
     "databricks/databricks-qwen3-next-80b-a3b-instruct": {
         "cache_creation_input_token_cost": 1.5001e-07,
@@ -61865,6 +61986,39 @@
         "supports_reasoning": true,
         "supports_vision": false
     },
+    "tencent/seedream-image-v5.0-flash": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.018,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://cloud.tencent.com/document/product/1823/130055",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
+    "tencent/seedream-image-v5.0-lite": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.035,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://www.tencentcloud.com/document/product/1300/78937",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
+    "tencent/seedream-image-v5.0-pro": {
+        "litellm_provider": "tencent",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.09,
+        "output_cost_per_image_token": 1.6e-06,
+        "source": "https://www.tencentcloud.com/document/product/1300/78937",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ]
+    },
     "cognition/swe-1.6": {
         "input_cost_per_token": 5e-07,
         "output_cost_per_token": 2.5e-06,
@@ -77774,7 +77928,11 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 5.25e-07,
+        "output_cost_per_token_priority": 2.1e-06,
+        "cache_read_input_token_cost_priority": 5.25e-08,
+        "cache_creation_input_token_cost_priority": 5.25e-07
     },
     "databricks/databricks-deepseek-v4-1-flash-1017": {
         "cache_creation_input_token_cost": 3.0002e-07,
@@ -77897,7 +78055,11 @@
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_video_input": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.50002e-06,
+        "output_cost_per_token_priority": 2.250003e-05,
+        "cache_read_input_token_cost_priority": 2.2498e-07,
+        "cache_creation_input_token_cost_priority": 4.50002e-06
     },
     "databricks/databricks-gpt-5-4-pro": {
         "cache_creation_input_token_cost": 2.999997e-05,
@@ -77959,7 +78121,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 1.99997e-05,
+        "cache_read_input_token_cost_priority": 1.9999e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-6-1-terra": {
         "cache_creation_input_token_cost": 1.24999e-06,
@@ -77991,7 +78161,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.99997e-06,
+        "output_cost_per_token_priority": 9.99999e-06,
+        "cache_read_input_token_cost_priority": 1.0003e-07,
+        "cache_creation_input_token_cost_priority": 2.49998e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 4.00001e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 1.500002e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06
     },
     "databricks/databricks-gpt-6-astra": {
         "cache_creation_input_token_cost": 1.249997e-05,
@@ -78023,7 +78201,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.999998e-05,
+        "output_cost_per_token_priority": 9.99999e-05,
+        "cache_read_input_token_cost_priority": 1.99997e-06,
+        "cache_creation_input_token_cost_priority": 2.500001e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 4.000003e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 0.0001500002,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05
     },
     "databricks/databricks-gpt-6-luna": {
         "cache_creation_input_token_cost": 1.2502e-07,
@@ -78055,7 +78241,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 1.9999e-07,
+        "output_cost_per_token_priority": 1.0003e-06,
+        "cache_read_input_token_cost_priority": 2.002e-08,
+        "cache_creation_input_token_cost_priority": 2.4997e-07,
+        "input_cost_per_token_above_272k_tokens_priority": 3.9998e-07,
+        "output_cost_per_token_above_272k_tokens_priority": 1.5001e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07
     },
     "databricks/databricks-gpt-6-sol": {
         "cache_creation_input_token_cost": 2.49998e-06,
@@ -78087,7 +78281,15 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "input_cost_per_token_priority": 4.00001e-06,
+        "output_cost_per_token_priority": 1.99997e-05,
+        "cache_read_input_token_cost_priority": 3.9998e-07,
+        "cache_creation_input_token_cost_priority": 5.00003e-06,
+        "input_cost_per_token_above_272k_tokens_priority": 8.00002e-06,
+        "output_cost_per_token_above_272k_tokens_priority": 2.99999e-05,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 8.0003e-07,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
     },
     "databricks/databricks-gpt-image-1": {
         "cache_creation_input_token_cost": 5.00003e-06,

provider_endpoints_support.json

@@ -3793,8 +3793,8 @@
         "responses": true,
         "text_completion": true,
         "embeddings": false,
-        "image_generations": false,
-        "image_edits": false,
+        "image_generations": true,
+        "image_edits": true,
         "audio_transcriptions": false,
         "audio_speech": false,
         "rerank": false,