Merge branch 'main' into litellm_semantic_cache_gate_hook

.agents/skills/database-access/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: database-access
 description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
+metadata:
+  internal: true
 ---
 
 # Database access

.github/CODEOWNERS

@@ -2,3 +2,4 @@
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /helm/ @yassin-berriai
 /terraform/ @yassin-berriai
+/.github/workflows/zizmor.yml @yuneng-berri

.github/workflows/cost-map-guard.yml

@@ -1,7 +1,7 @@
 name: Cost map guard
 
-on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's cost map files are read as data and never executed
-  pull_request_target:
+on:
+  pull_request:
     branches:
       - main
       - "litellm_**"
@@ -19,6 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
+          ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
       - name: Fetch the pull request head and its merge base
         id: revisions

.github/workflows/interactions-spec-canary.yml

@@ -19,7 +19,7 @@ jobs:
     name: interactions-spec-canary
     if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
     permissions:
       contents: read
     outputs:
@@ -39,8 +39,11 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        uses: ./.github/actions/rust-bridge
+
       - name: Install dependencies
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
 
       - name: Run the compliance tests against Google's live spec
         id: compliance

.github/workflows/spend-logs-guard.yml

@@ -1,13 +1,9 @@
 name: Spend logs guard
 
-on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed
-  pull_request_target:
+on:
+  pull_request:
     branches:
       - main
-  pull_request_review:
-    types:
-      - submitted
-      - dismissed
 
 permissions:
   contents: read
@@ -23,7 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          ref: ${{ github.event.pull_request.base.ref }}
+          ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
       - name: Fetch the pull request head, its merge base and its reviews
         id: revisions

.github/workflows/test-linting.yml

@@ -561,6 +561,9 @@ jobs:
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 
+      - name: router_surface_check
+        run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py
+
       - name: test_chat_completion_imports
         run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
 

.github/workflows/test-merge-smoke.yml

@@ -132,7 +132,7 @@ jobs:
   windows-sdk:
     name: windows-sdk
     runs-on: windows-2025
-    timeout-minutes: 10
+    timeout-minutes: 30
     env:
       LITELLM_LOCAL_MODEL_COST_MAP: "True"
     defaults:
@@ -154,8 +154,11 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        uses: ./.github/actions/rust-bridge
+
       - name: Install dependencies
-        run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --group dev --python 3.11
+        run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
 
       - name: Run the Windows tests
         run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests

.github/workflows/test-unit.yml

@@ -137,6 +137,7 @@ jobs:
         include:
           - shard: core-utils
             test-path: |-
+              tests/unit/accounting
               tests/unit/batch_completion
               tests/unit/decisions
               tests/unit/extraction
@@ -244,6 +245,7 @@ jobs:
               tests/unit/llms/clarifai
               tests/unit/llms/claude_code
               tests/unit/llms/cloudflare
+              tests/unit/llms/codestral
               tests/unit/llms/codex
               tests/unit/llms/cohere
               tests/unit/llms/cometapi
@@ -1306,6 +1308,8 @@ jobs:
           done
   openai-sdk-compat:
     name: OpenAI ${{ matrix.openai-version }} compatibility
+    needs: rust-bridge
+    if: ${{ !cancelled() }}
     permissions:
       contents: read
       pull-requests: read
@@ -1342,10 +1346,18 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Set up the Rust bridge
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/rust-bridge
+        with:
+          artifact: ${{ needs.rust-bridge.outputs.artifact }}
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group dev --group proxy-dev --extra proxy --no-install-workspace
+          cp rust-bridge-dist/litellm/rust_bridge/_native.* litellm/rust_bridge/
           uv pip install "openai==${OPENAI_VERSION:?}" --exclude-newer-package openai=2026-10-07T00:00:00Z
           uv run --no-sync python -c 'import os, openai; assert openai.__version__ == os.environ["OPENAI_VERSION"], openai.__version__'
 

.github/workflows/zizmor.yml

@@ -27,10 +27,15 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Install zizmor
+        run: |
+          curl -fsSL --retry 3 -o "$RUNNER_TEMP/zizmor.tar.gz" \
+            https://github.com/zizmorcore/zizmor/releases/download/v1.31.1/zizmor-x86_64-unknown-linux-gnu.tar.gz
+          echo "d185f726bdaf92df78273217dfdd24ebcd6b0ef48cf3ce776ff5762b6e138c99  $RUNNER_TEMP/zizmor.tar.gz" | sha256sum -c -
+          tar xzf "$RUNNER_TEMP/zizmor.tar.gz" -C "$RUNNER_TEMP" zizmor
+
       - name: Run zizmor
-        uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
-        with:
-          version: "1.24.1"
-          min-severity: medium
-          advanced-security: false
-          annotations: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          "$RUNNER_TEMP/zizmor" --no-ignores --no-config --min-severity=medium --format=github .

README.md

@@ -427,6 +427,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) |  |  |  | ✅ |  |  |  |  |  |  |
+| [ZeroGPU (`zerogpu`)](https://docs.litellm.ai/docs/providers/zerogpu) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 

ci_cd/spend_logs_guard.py

@@ -2,9 +2,12 @@
 needs an approving review from the spend-log owner before it can merge.
 
 CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
-so a path rule would put every migration on the owner. This gate reads the diff instead: when no
-added line in the watched files mentions the table it passes at once, otherwise it passes only once
-the owner's latest review on the pull request is an approval. The workflow re-runs on every review.
+so a path rule would put every migration on the owner. This gate reads the diff instead: a schema.prisma
+hunk counts when it adds or removes a line inside the `model LiteLLM_SpendLogs { ... }` block, a migration
+hunk counts when an added line names the table. Without such a hunk it passes at once, otherwise it passes
+only once the owner's latest review on the pull request is an approval that still covers every spend-log line
+in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never saw).
+Reviews do not trigger the workflow, so the check is re-run by hand after approving.
 """
 
 from __future__ import annotations
@@ -13,8 +16,10 @@ import argparse
 import re
 import subprocess
 import sys
-from collections.abc import Sequence
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
 from pathlib import Path
+from types import MappingProxyType
 from typing import Final
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter
@@ -25,7 +30,10 @@ WATCHED: Final = (
     "litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
 )
 MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
-TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs", re.IGNORECASE)
+TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
+DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
+MODEL_START: Final = re.compile(r"^model\s+LiteLLM_SpendLogs\s*\{")
+HUNK: Final = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@[^\n]*\n", re.MULTILINE)
 
 
 class Reviewer(BaseModel):
@@ -37,31 +45,77 @@ class Review(BaseModel):
     model_config = ConfigDict(frozen=True, extra="ignore")
     state: str
     user: Reviewer
+    commit_id: str | None = None
 
 
 _REVIEWS: Final = TypeAdapter(list[Review])
 
 
+@dataclass(frozen=True, slots=True)
+class Block:
+    """Lines of `model LiteLLM_SpendLogs { ... }` in one schema.prisma copy at the merge base and at the head."""
+
+    base: range
+    head: range
+
+
+@dataclass(frozen=True, slots=True)
+class Hunk:
+    removed: tuple[tuple[int, str], ...]
+    added: tuple[tuple[int, str], ...]
+
+
 def watched(path: str) -> bool:
     return path in WATCHED or (path.startswith(MIGRATIONS_PREFIX) and path.endswith("/migration.sql"))
 
 
-def _added_lines(section: str) -> tuple[str, ...]:
-    body: Final = section.split("\n@@", 1)
-    return tuple(line[1:] for line in body[1].splitlines() if line.startswith("+")) if len(body) == 2 else ()
+def model_block(schema: str) -> range:
+    """1-based line range of `model LiteLLM_SpendLogs { ... }` including both braces, empty when the schema has none."""
+    lines: Final = schema.splitlines()
+    starts: Final = tuple(number for number, line in enumerate(lines, 1) if MODEL_START.match(line))
+    if not starts:
+        return range(0)
+    ends: Final = tuple(number for number, line in enumerate(lines, 1) if number > starts[0] and line.startswith("}"))
+    return range(starts[0], (ends[0] if ends else len(lines)) + 1)
 
 
-def spend_log_hits(diff: str) -> tuple[str, ...]:
-    """`path: added line` for every added line in a watched file that names the table."""
+def _hunks(section: str) -> tuple[Hunk, ...]:
+    headers: Final = tuple(HUNK.finditer(section))
+    ends: Final = tuple(header.start() for header in headers[1:]) + (len(section),)
+    return tuple(_hunk(header, section[header.end() : end]) for header, end in zip(headers, ends))
+
+
+def _hunk(header: re.Match[str], body: str) -> Hunk:
+    lines: Final = body.splitlines()
+    return Hunk(
+        removed=tuple(enumerate((line[1:] for line in lines if line.startswith("-")), int(header.group(1)))),
+        added=tuple(enumerate((line[1:] for line in lines if line.startswith("+")), int(header.group(2)))),
+    )
+
+
+def spend_log_hits(diff: str, blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    """`path: +line` or `path: -line` for every changed line that touches the table.
+
+    A schema.prisma line counts when its number falls inside that copy's `blocks` entry (head for added lines,
+    merge base for removed ones), a migration line when it is added and names the table."""
     sections: Final = re.split(r"^diff --git [^\n]*\n", diff, flags=re.MULTILINE)
-    return tuple(f"{path}: {line.strip()}" for path, line in _watched_added_lines(sections) if TABLE.search(line))
-
-
-def _watched_added_lines(sections: Sequence[str]) -> tuple[tuple[str, str], ...]:
-    pairs: Final = tuple((_path_of(section), section) for section in sections)
+    paths: Final = tuple((_path_of(section), section) for section in sections)
     return tuple(
-        (path, line) for path, section in pairs if watched(path) for line in _added_lines(section)
-    )  # comprehension-ok: path then its lines
+        hit for path, section in paths if watched(path) for hit in _section_hits(path, _hunks(section), blocks)
+    )  # comprehension-ok: path then its hits
+
+
+def _section_hits(path: str, hunks: Sequence[Hunk], blocks: Mapping[str, Block]) -> tuple[str, ...]:
+    if path not in blocks:
+        return tuple(f"{path}: +{line.strip()}" for hunk in hunks for _, line in hunk.added if TABLE.search(line))
+    block: Final = blocks[path]
+    removed: Final = tuple(
+        f"{path}: -{line.strip()}" for hunk in hunks for number, line in hunk.removed if number in block.base
+    )  # comprehension-ok: hunk then its lines
+    added: Final = tuple(
+        f"{path}: +{line.strip()}" for hunk in hunks for number, line in hunk.added if number in block.head
+    )  # comprehension-ok: hunk then its lines
+    return removed + added
 
 
 def _path_of(section: str) -> str:
@@ -70,11 +124,33 @@ def _path_of(section: str) -> str:
 
 
 def approved_by(reviews: Sequence[Review], login: str) -> bool:
-    """True when `login`'s most recent review on the pull request is an approval."""
-    states: Final = tuple(review.state for review in reviews if review.user.login == login)
+    """True when `login`'s most recent review that changes GitHub's approval state is an approval.
+
+    COMMENTED and PENDING reviews leave an earlier approval in force on GitHub, so they are skipped here too.
+    """
+    own: Final = tuple(review.state for review in reviews if review.user.login == login)
+    states: Final = tuple(state for state in own if state in DECIDING_STATES)
     return bool(states) and states[-1] == "APPROVED"
 
 
+def latest_approval(reviews: Sequence[Review], login: str) -> Review | None:
+    """`login`'s most recent APPROVED review on the pull request, the one GitHub keeps in force after pushes."""
+    own: Final = tuple(review for review in reviews if review.user.login == login and review.state == "APPROVED")
+    return own[-1] if own else None
+
+
+def approval_covers(
+    approval: Review, head: str, hits: Sequence[str], hits_at: Callable[[str], Sequence[str] | None]
+) -> bool:
+    """True when the approval is on the head, or the head adds no spend-log line the approved commit did not."""
+    if approval.commit_id is None:
+        return False
+    if approval.commit_id == head:
+        return True
+    reviewed: Final = hits_at(approval.commit_id)
+    return reviewed is not None and set(hits) <= set(reviewed)
+
+
 def added_diff(base: str, head: str) -> str:
     result: Final = subprocess.run(
         ("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX),
@@ -85,6 +161,27 @@ def added_diff(base: str, head: str) -> str:
     return result.stdout
 
 
+def file_at(revision: str, path: str) -> str:
+    """Contents of `path` at `revision`, empty when the file does not exist there."""
+    result: Final = subprocess.run(("git", "show", f"{revision}:{path}"), check=False, capture_output=True, text=True)
+    return result.stdout if result.returncode == 0 else ""
+
+
+def schema_blocks(base: str, head: str) -> Mapping[str, Block]:
+    return MappingProxyType(
+        {path: Block(base=model_block(file_at(base, path)), head=model_block(file_at(head, path))) for path in WATCHED}
+    )
+
+
+def fetched(commit: str) -> bool:
+    result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
+    return result.returncode == 0
+
+
+def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
+    return spend_log_hits(added_diff(base, commit), schema_blocks(base, commit)) if fetched(commit) else None
+
+
 class Arguments(BaseModel):
     model_config = ConfigDict(frozen=True)
     base: str
@@ -100,18 +197,28 @@ def main(argv: Sequence[str]) -> int:
     parser.add_argument("--approver", required=True, help="GitHub login whose approval unlocks the merge")
     parser.add_argument("--reviews", required=True, help="JSON file with the pull request's reviews")
     args: Final = Arguments.model_validate(vars(parser.parse_args(argv)))
-    hits: Final = spend_log_hits(added_diff(args.base, args.head))
+    hits: Final = spend_log_hits(added_diff(args.base, args.head), schema_blocks(args.base, args.head))
     if not hits:
-        sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
+        sys.stdout.write(
+            "No change inside model LiteLLM_SpendLogs in schema.prisma and no added migration line names it; "
+            "nothing to review.\n"
+        )
         return 0
-    if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver):
+    reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
+    approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None
+    if approval is not None and approval_covers(
+        approval, args.head, hits, lambda commit: reviewed_hits(args.base, commit)
+    ):
         sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n")
         return 0
     listed: Final = "".join(f"  {hit}\n" for hit in hits)
-    sys.stdout.write(
+    reason: Final = (
         f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n"
-        f"{listed}The check re-runs when a review is submitted.\n"
+        if approval is None
+        else f"LiteLLM_SpendLogs lines changed after @{args.approver} approved {approval.commit_id}, "
+        f"a new approving review is needed:\n"
     )
+    sys.stdout.write(f"{reason}{listed}After approving, re-run this job.\n")
     return 1
 
 

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -32,6 +32,7 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
+from litellm.llms.base_llm.guardrail_translation.utils import effective_skip_tool_message_for_guardrail
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails._content_utils import iter_tool_json_fields, map_tool_json_text, walk_user_text
 from litellm.types.guardrails import GuardrailEventHooks
@@ -753,7 +754,12 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         _masked_entity_count.set({})
 
         # Covers multimodal list content + Responses-API input.
-        inspected = walk_user_text(data, self.redact_text, self.redact_tool_payload) + self._redact_prompt(data)
+        inspected = walk_user_text(
+            data,
+            self.redact_text,
+            self.redact_tool_payload,
+            skip_tool_message=effective_skip_tool_message_for_guardrail(self),
+        ) + self._redact_prompt(data)
 
         if inspected == 0:
             # Image-only, empty-text, and unsupported payloads inspected

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -28,7 +28,7 @@ import httpx
 from fastapi import HTTPException
 from openai import APIConnectionError, AsyncOpenAI
 from openai.types.file_deleted import FileDeleted
-from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, Unpack
 
 import litellm
@@ -53,7 +53,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_metadata,
 )
 from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
-from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
+from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
 from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
     build_owner_filter,
@@ -309,6 +309,7 @@ def _is_xai_batch_output(
 class _ManagedFileRow(Protocol):
     unified_file_id: str
     file_object: OpenAIFileObject
+    model_mappings: Mapping[str, str]
     flat_model_file_ids: Sequence[str]
     storage_backend: Optional[str]
     storage_url: Optional[str]
@@ -362,6 +363,8 @@ class _ManagedResourceDatabase(Protocol):
     @property
     def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
 
+    async def query_raw(self, query: str, *args: object) -> object: ...
+
     @property
     def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
 
@@ -385,10 +388,83 @@ class _RouterFileCallKwargs(TypedDict, total=False):
     custom_llm_provider: ReadOnly[str | None]
 
 
+def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
+    return prisma_client.db
+
+
 def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedFileTableActions:
     return prisma_client.db.litellm_managedfiletable
 
 
+class _ListedFileRow(Protocol):
+    @property
+    def unified_file_id(self) -> str: ...
+
+    @property
+    def file_object(self) -> object: ...
+
+    @property
+    def model_mappings(self) -> Mapping[str, object]: ...
+
+    @property
+    def storage_backend(self) -> str | None: ...
+
+
+class _ScopedFileRow(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    unified_file_id: str
+    file_object: JsonValue = None
+    model_mappings: Mapping[str, JsonValue]
+    storage_backend: str | None = None
+
+
+_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...])
+
+
+def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]:
+    match owner_filter:
+        case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}:
+            return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id)
+        case {"created_by": str() as user_id}:
+            return '"created_by" = $1', (user_id,)
+        case {"team_id": str() as team_id}:
+            return '"team_id" = $1', (team_id,)
+        case _:
+            return ("FALSE", ()) if owner_filter else ("TRUE", ())
+
+
+async def _scoped_file_rows(
+    database: _ManagedResourceDatabase,
+    owner_filter: Mapping[str, object],
+    model_mapping_keys: frozenset[str],
+    cursor_id: str | None,
+    direction: Literal["asc", "desc"],
+    take: int,
+) -> tuple[_ScopedFileRow, ...]:
+    owner_condition, owner_params = _owner_condition(owner_filter)
+    keys_param: Final = len(owner_params) + 1
+    comparison: Final = "<" if direction == "desc" else ">"
+    cursor_condition: Final = (
+        f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" '
+        f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) '
+        if cursor_id
+        else ""
+    )
+    order: Final = direction.upper()
+    rows: Final = await database.query_raw(
+        'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" '
+        f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) '
+        f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} '
+        f"LIMIT ${keys_param + (2 if cursor_id else 1)}",
+        *owner_params,
+        sorted(model_mapping_keys),
+        *((cursor_id,) if cursor_id else ()),
+        take,
+    )
+    return _SCOPED_FILE_ROWS.validate_python(rows)
+
+
 def _iter_provider_file_id_pairs(
     rows: Sequence[_ManagedFileRow],
     requested_provider_file_ids: frozenset[str],
@@ -2265,14 +2341,23 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         user_api_key_dict: UserAPIKeyAuth,
         limit: Optional[int] = None,
         after: Optional[str] = None,
+        model_scope: Optional[FileListModelScope] = None,
+        before: Optional[str] = None,
         **data: Dict,
     ) -> FileListPage:
         """List the managed files the caller owns, newest first.
 
         Pagination is keyset based on ``unified_file_id`` so a key that owns
         every file on the proxy still reads one bounded page at a time.
-        ``purpose`` is applied after parsing, because the managed file table
-        keeps it inside the ``file_object`` blob instead of a column, and rows
+        ``after`` pages toward older files and ``before`` toward newer ones,
+        each page still ordered newest first.
+        ``model_scope`` is applied in the query, so only rows whose
+        ``model_mappings`` name one of its deployment ids or model names, and
+        storage-backed rows, leave the database; a storage-backed row then
+        stays only when its model name is one the router sends to the listed
+        provider. ``purpose`` is applied after parsing, because the managed
+        file table keeps it inside the ``file_object`` blob instead of a
+        column. Rows
         whose blob will not parse drop out there too, so a chunk of rows can
         yield fewer matches than the page holds. Successive chunks are read
         until the page is full or the caller's rows run out, which keeps
@@ -2281,9 +2366,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         scan has to continue past it, the chunk widens to
         ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
         that many rows instead of one per page. That bound is per query, not
-        per request: the work is still linear in the rows the caller owns, and
-        a filter matching nothing reads every one of them, with no index
-        covering either the owner filter or the sort.
+        per request: a ``purpose`` filter matching nothing still reads every
+        row that passes the query.
         """
         validate_file_list_limit(limit)
         validate_file_list_purpose(purpose)
@@ -2292,36 +2376,60 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         if owner_filter is None:
             return FileListPage.model_validate(build_list_page([]))
 
-        if after:
-            cursor_row = await _managed_file_table(self.prisma_client).find_first(
-                where={**owner_filter, "unified_file_id": after}
+        if after and before:
+            raise ProxyException(
+                message="Pass either 'after' (or 'after_id') or 'before_id' when listing files, not both.",
+                type="invalid_request_error",
+                param="before_id",
+                code=400,
+                openai_code="invalid_value",
+            )
+        cursor_param: Final = "before_id" if before else "after"
+        cursor: Final = before or after
+        database: Final = _managed_database(self.prisma_client)
+        if cursor:
+            cursor_row = await database.litellm_managedfiletable.find_first(
+                where={**owner_filter, "unified_file_id": cursor}
             )
             if cursor_row is None:
                 raise ProxyException(
-                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    message=f"Invalid '{cursor_param}' cursor: no file found with id '{cursor}'.",
                     type="invalid_request_error",
-                    param="after",
+                    param=cursor_param,
                     code=400,
                     openai_code="invalid_value",
                 )
 
+        if model_scope is not None and not model_scope.model_mapping_keys:
+            return FileListPage.model_validate(build_list_page([]))
+
         page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
         matches: Final[List[OpenAIFileObject]] = []
-        cursor_id = after
+        cursor_id = cursor
         chunk_size = page_size + 1
+        direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc"
 
         while len(matches) <= page_size:
             cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
-            chunk = await _managed_file_table(self.prisma_client).find_many(
-                where=owner_filter,
-                take=chunk_size,
-                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
-                **cursor_args,
+            chunk: Sequence[_ListedFileRow] = (
+                await _scoped_file_rows(
+                    database, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size
+                )
+                if model_scope is not None
+                else await database.litellm_managedfiletable.find_many(
+                    where=owner_filter,
+                    take=chunk_size,
+                    order=[{"created_at": direction}, {"unified_file_id": direction}],
+                    **cursor_args,
+                )
             )
             matches.extend(
                 _public_file_object(parsed_file_object, row.unified_file_id)
                 for row in chunk
-                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                if (
+                    model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
+                )
+                and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
                 and (purpose is None or parsed_file_object.purpose == purpose)
             )
             if len(chunk) < chunk_size:
@@ -2329,7 +2437,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             cursor_id = chunk[-1].unified_file_id
             chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
 
-        return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
+        page: Final = matches[:page_size]
+        return FileListPage.model_validate(
+            build_list_page(page[::-1] if before else page, has_more=len(matches) > page_size)
+        )
 
     def _is_batch_polling_enabled(self) -> bool:
         """

gateway/Dockerfile

@@ -153,7 +153,7 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
     python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
-    python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
+    python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__"
 
 USER nonroot
 

litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py

@@ -343,11 +343,16 @@ def btree_max_entry_bytes(block_size: int) -> int:
 
 
 def _value_bytes(column: ColumnType, identifier: "sql.Identifier") -> "sql.Composed":
+    """The bytes a value takes in the index entry: its length, or its stored size when
+    Postgres compressed it, since `index_form_tuple` keeps the compressed bytes."""
     from psycopg import sql
 
-    if column.type_name in _OCTET_LENGTH_TYPES:
-        return sql.SQL("octet_length({})").format(identifier)
-    return sql.SQL("octet_length({}::text)").format(identifier)
+    length: Final = (
+        sql.SQL("octet_length({})").format(identifier)
+        if column.type_name in _OCTET_LENGTH_TYPES
+        else sql.SQL("octet_length({}::text)").format(identifier)
+    )
+    return sql.SQL("least({}, pg_column_size({}))").format(length, identifier)
 
 
 def _aligned(offset: "sql.Composable", align: str) -> "sql.Composed":

litellm-proxy-extras/litellm_proxy_extras/migrations/20261021000000_managed_file_created_by_created_at_index/migration.sql

@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_ManagedFileTable_created_by_created_at_idx"
+ON "LiteLLM_ManagedFileTable" ("created_by", "created_at" DESC);

litellm-proxy-extras/litellm_proxy_extras/migrations/20261027000000_add_accounting_ledger_tables/migration.sql

@@ -0,0 +1,47 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE TYPE "AccountingEventKind" AS ENUM ('inbound_usage', 'outbound_usage', 'cost');
+
+CREATE TYPE "AccountingDeliveryState" AS ENUM ('pending', 'applied', 'parked');
+
+CREATE TYPE "AccountingDeliveryTarget" AS ENUM ('LiteLLM_VerificationToken', 'LiteLLM_UserTable', 'LiteLLM_TeamTable', 'LiteLLM_TeamMembership', 'LiteLLM_OrganizationTable', 'LiteLLM_OrganizationMembership', 'LiteLLM_EndUserTable', 'LiteLLM_AgentsTable', 'LiteLLM_TagTable', 'LiteLLM_ProjectTable', 'LiteLLM_ModelAccessGroupBudgetTable', 'LiteLLM_BudgetTable', 'LiteLLM_BudgetWindowSpend', 'LiteLLM_DailyUserSpend', 'LiteLLM_DailyTeamSpend', 'LiteLLM_DailyOrganizationSpend', 'LiteLLM_DailyEndUserSpend', 'LiteLLM_DailyAgentSpend', 'LiteLLM_DailyTagSpend', 'LiteLLM_DailyModelUsage', 'LiteLLM_DailyToolSpend', 'LiteLLM_AutoRouterDailySpend');
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingEvent" (
+    "event_id" TEXT NOT NULL,
+    "kind" "AccountingEventKind" NOT NULL,
+    "source_event_id" TEXT,
+    "schema_version" INTEGER NOT NULL,
+    "payload_hash" TEXT NOT NULL,
+    "occurred_at" TIMESTAMP(3) NOT NULL,
+    "recorded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "payload" JSONB NOT NULL,
+
+    CONSTRAINT "LiteLLM_AccountingEvent_schema_version_check" CHECK ("schema_version" > 0),
+    CONSTRAINT "LiteLLM_AccountingEvent_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingDelivery" (
+    "event_id" TEXT NOT NULL,
+    "target" "AccountingDeliveryTarget" NOT NULL,
+    "version" INTEGER NOT NULL,
+    "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "applied_at" TIMESTAMP(3),
+    "last_error" TEXT,
+
+    CONSTRAINT "LiteLLM_AccountingDelivery_version_check" CHECK ("version" > 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_attempts_check" CHECK ("attempts" >= 0),
+    CONSTRAINT "LiteLLM_AccountingDelivery_pkey" PRIMARY KEY ("event_id","target","version")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_recorded_at_idx" ON "LiteLLM_AccountingEvent"("recorded_at");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_source_event_id_idx" ON "LiteLLM_AccountingEvent"("kind", "source_event_id");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingDelivery_target_state_next_attempt_at_idx" ON "LiteLLM_AccountingDelivery"("target", "state", "next_attempt_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
   @@index([unified_file_id])
   @@index([flat_model_file_ids], type: Gin)
   @@index([team_id, created_at(sort: Desc)])
+  @@index([created_by, created_at(sort: Desc)])
 }
 
 model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
   @@index([window_end])
   @@index([created_at])
 }
+
+enum AccountingEventKind {
+  inbound_usage
+  outbound_usage
+  cost
+}
+
+enum AccountingDeliveryState {
+  pending
+  applied
+  parked
+}
+
+enum AccountingDeliveryTarget {
+  LiteLLM_VerificationToken
+  LiteLLM_UserTable
+  LiteLLM_TeamTable
+  LiteLLM_TeamMembership
+  LiteLLM_OrganizationTable
+  LiteLLM_OrganizationMembership
+  LiteLLM_EndUserTable
+  LiteLLM_AgentsTable
+  LiteLLM_TagTable
+  LiteLLM_ProjectTable
+  LiteLLM_ModelAccessGroupBudgetTable
+  LiteLLM_BudgetTable
+  LiteLLM_BudgetWindowSpend
+  LiteLLM_DailyUserSpend
+  LiteLLM_DailyTeamSpend
+  LiteLLM_DailyOrganizationSpend
+  LiteLLM_DailyEndUserSpend
+  LiteLLM_DailyAgentSpend
+  LiteLLM_DailyTagSpend
+  LiteLLM_DailyModelUsage
+  LiteLLM_DailyToolSpend
+  LiteLLM_AutoRouterDailySpend
+}
+
+// Durable accounting ledger. One row per usage or cost event; payload holds the
+// private event envelope versioned by schema_version. Usage rows retain request-scoped
+// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
+model LiteLLM_AccountingEvent {
+  event_id        String              @id
+  kind            AccountingEventKind
+  source_event_id String?
+  schema_version  Int
+  payload_hash    String
+  occurred_at     DateTime
+  recorded_at     DateTime            @default(now())
+  payload         Json
+
+  @@index([recorded_at])
+  @@index([kind, source_event_id])
+}
+
+// One row per (event, target table, version): pending work for the projector and,
+// once applied in the same transaction as the target write, the receipt that stops a retry.
+model LiteLLM_AccountingDelivery {
+  event_id        String
+  target          AccountingDeliveryTarget
+  version         Int
+  state           AccountingDeliveryState  @default(pending)
+  attempts        Int                      @default(0)
+  next_attempt_at DateTime                 @default(now())
+  applied_at      DateTime?
+  last_error      String?
+
+  @@id([event_id, target, version])
+  @@index([target, state, next_attempt_at])
+}

litellm-rust/.agents/skills/rust-error/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-error
 description: Define, wrap, convert, classify, report or test Rust error types in litellm-rust with thiserror. Use when adding an error type or variant, mapping one crate's error into another, turning an error into a gateway response or Python exception, deciding retry or fault from an error, or replacing call sites that build errors from strings
+metadata:
+  internal: true
 ---
 
 # Rust errors

litellm-rust/.agents/skills/rust-gateway/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-gateway
 description: Add or change an HTTP route, extractor, response envelope, stream or router mount in the litellm-rust gateway crates (crates/gateway and crates/gateway-*). Use when adding an endpoint, parsing a request body, choosing an error envelope for an API surface, relaying SSE, or writing a router-level test
+metadata:
+  internal: true
 ---
 
 # Rust gateway

litellm-rust/.agents/skills/rust-string-enums/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-string-enums
 description: Define or refactor Rust string-valued enums and their Serde adapters in litellm-rust, using Strum and serde_with while preserving parsing, wire values, and schemas
+metadata:
+  internal: true
 ---
 
 # Rust string enums

litellm-rust/.agents/skills/rust-tracing/SKILL.md

@@ -1,6 +1,8 @@
 ---
 name: rust-tracing
 description: Add or change Rust diagnostic tracing in litellm-rust, including route spans, subscriber layers, and Python logger delivery
+metadata:
+  internal: true
 ---
 
 # Rust tracing

litellm-rust/Cargo.lock

@@ -4393,6 +4393,10 @@ dependencies = [
  "rstest",
 ]
 
+[[package]]
+name = "litellm-router-core"
+version = "0.1.0"
+
 [[package]]
 name = "litellm-router-types"
 version = "0.1.0"

litellm-rust/crates/core-utils/src/core_helpers.rs

@@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
     ("max_tokens", "length"),
     ("refusal", "content_filter"),
     ("compaction", "length"),
+    ("model_context_window_exceeded", "length"),
     ("guardrail_intervened", "content_filter"),
     ("content_filtered", "content_filter"),
     ("content_filter", "content_filter"),
@@ -74,6 +75,7 @@ mod tests {
     #[case::end_turn("end_turn", "stop")]
     #[case::stop_sequence("stop_sequence", "stop")]
     #[case::max_tokens("max_tokens", "length")]
+    #[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
     #[case::refusal("refusal", "content_filter")]
     #[case::guardrail_intervened("guardrail_intervened", "content_filter")]
     #[case::content_filtered("content_filtered", "content_filter")]

litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs

@@ -113,6 +113,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
     "meta",
     "cognition",
     "scx-ai",
+    "zerogpu",
 ];
 
 impl ExceptionFamily {

litellm-rust/crates/model-catalog/src/lib.rs

@@ -1,3 +1,36 @@
+//! The model catalog: per-model pricing, context limits and capabilities.
+//!
+//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as
+//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model
+//! cost and what can it do"; it knows nothing about configured deployments. The router's view of
+//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined
+//! in `litellm-router-core`, which reads this catalog through its dynamic config.
+//!
+//! # Structure
+//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where
+//!   the body came from: the bundled copy in the repo or the remote copy, with revision and etag),
+//!   and keeps every entry in file order with its raw fields.
+//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
+//!   never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`,
+//!   modalities and modes in `capabilities`).
+//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
+//!   found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of
+//!   failing the parse.
+//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
+//!   families. They are data in the file, not entries.
+//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that
+//!   lost most of its models is refused), and `validate_registry` checks every entry against the
+//!   checked-in contract.
+//!
+//! # Lookups
+//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
+//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider
+//! prefixes are not stripped here; callers decide which keys to try.
+//!
+//! # Readers
+//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
+//! performs I/O; loading and refreshing the file is the caller's job.
+
 mod capabilities;
 mod catalog;
 mod error;

litellm-rust/crates/python-bridge/src/logger/tests.rs

@@ -187,6 +187,7 @@ fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication
         locals
             .set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
             .unwrap();
+        locals.set_item("native", crate::native_module(py)).unwrap();
         let importable = py
             .eval(
                 c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
@@ -205,6 +206,7 @@ import asyncio
 import logging
 import sys
 sys.path.insert(0, repo_root)
+sys.modules['litellm.rust_bridge._native'] = native
 import litellm
 from litellm._logging import verbose_logger, session_id_var, trace_id_var
 

litellm-rust/crates/router-core/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "litellm-router-core"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
+
+[dependencies]

litellm-rust/crates/router-core/src/lib.rs

@@ -0,0 +1,63 @@
+//! Routing for LiteLLM: decide which deployment serves a request, and drive retries, fallbacks
+//! and cooldowns around the attempts.
+//!
+//! This crate defines the boundary `litellm.Router` is moving behind. Today the Python router in
+//! `litellm/router.py` implements all of it; this crate is where the Rust core will implement the
+//! same interface. Until then it holds this overview and nothing else, so the interface is written
+//! down in code before either side changes.
+//!
+//! # The public surface
+//! Code outside the router uses `litellm.Router` through two kinds of members and nothing else:
+//! - The endpoint methods (`acompletion`, `aembedding`, `aresponses`, `aanthropic_messages`, the
+//!   files, batches, assistants and vector store wrappers, 37 in all). Each is a thin wrapper over
+//!   `call(op, request)`.
+//! - Eight entry points: `registry()`, `settings()`, `health()`, `select()`, `apply()`,
+//!   `observe()`, `register_plugin()` and `retire()`.
+//!
+//! Every other member that outside code uses today has a named replacement among those. The old
+//! names stay available for about a month through a `DeprecatedRouterMembers` mixin, which
+//! implements them over the entry points and warns on use. A CI check
+//! (`tests/code_coverage_tests/router_surface_check.py`) keeps an allowlist of the members outside
+//! code touches; the list may only shrink.
+//!
+//! # Structure
+//! - `Registry`: the configured deployments, indexed by id, model group, alias, wildcard pattern
+//!   and team. Immutable and versioned. Seven queries: `deployment(id)`, `deployments(filter)`,
+//!   `names(filter)`, `resolve(name, scope)`, `group(name)`, `access_groups()` and
+//!   `credentials(id)`. "Registry" is the router's view of what is configured; per-model pricing
+//!   and capabilities are the model catalog (`litellm-model-catalog`), which the router reads
+//!   through its dynamic config.
+//! - `RouterConfig` / `RouterDynamicConfig`: everything the router needs to know, passed in. The
+//!   router never reads process globals. Static config (model list, strategy, retry, timeout and
+//!   cooldown policy, Redis settings, plugins) is fixed for the router's life. Dynamic config
+//!   (pricing, model aliases, fallbacks, retry counts, callbacks) is a versioned snapshot read
+//!   once per call.
+//! - `Settings`: one typed, frozen view of the configuration a caller may read.
+//! - `Health`: cooldowns, blocked deployments and usage, kept in shared cache keys so Python and
+//!   Rust workers see the same state. `report(id, outcome)` records outcomes the router did not
+//!   see itself (a health check, an interceptor that marks a deployment failed).
+//! - `select(model, request)`: resolve a name, filter eligible deployments, pick one with the
+//!   configured strategy, without calling it.
+//! - `apply(changes)`: one `ChangeSet` (upserts, deletes, settings), applied atomically, producing
+//!   a new registry version.
+//! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the
+//!   policy is exhausted.
+//!
+//! # Execution
+//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs
+//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A
+//! started attempt is never replayed on a different executor.
+//!
+//! # Hosts
+//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select`
+//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks,
+//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all
+//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost`
+//! serves them from the existing Python code when the router runs inside Python; `NativeHost`
+//! serves them in the Rust server, where an operation without a Rust implementation makes the
+//! config fail at build time, never partway through a request.
+//!
+//! # Plugins
+//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
+//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
+//! the current request), never the router itself.

litellm/__init__.py

@@ -681,6 +681,7 @@ moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
 openinfer_models: Final[Set[str]] = set()
+zerogpu_models: Final[Set[str]] = set()
 v0_models: Set = set()
 morph_models: Set = set()
 opencode_go_models: Final[set[str]] = set()  # mutable-ok: populated when the shared model catalog loads
@@ -969,6 +970,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             darkbloom_models.add(key)
         elif value.get("litellm_provider") == "openinfer":
             openinfer_models.add(key)
+        elif value.get("litellm_provider") == "zerogpu":
+            zerogpu_models.add(key)
         elif value.get("litellm_provider") == "v0":
             v0_models.add(key)
         elif value.get("litellm_provider") == "morph":
@@ -1142,6 +1145,7 @@ model_list = list(
     | publicai_models
     | darkbloom_models
     | openinfer_models
+    | zerogpu_models
     | v0_models
     | morph_models
     | opencode_go_models
@@ -1261,6 +1265,7 @@ def _build_models_by_provider() -> dict:
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
         "openinfer": openinfer_models,
+        "zerogpu": zerogpu_models,
         "v0": v0_models,
         "morph": morph_models,
         "opencode_go": opencode_go_models,
@@ -1331,7 +1336,6 @@ all_embedding_models = (
 )
 
 ####### IMAGE GENERATION MODELS ###################
-openai_image_generation_models = ["dall-e-2", "dall-e-3"]
 
 ####### VIDEO GENERATION MODELS ###################
 openai_video_generation_models = ["sora-2"]
@@ -1739,6 +1743,9 @@ if TYPE_CHECKING:
     from .llms.lm_studio.decisions.transformation import (
         LMStudioDecisionsConfig as LMStudioDecisionsConfig,
     )
+    from .llms.ollama.decisions.transformation import (
+        OllamaDecisionsConfig as OllamaDecisionsConfig,
+    )
     from .llms.laya.decisions.transformation import (
         LayaDecisionsConfig as LayaDecisionsConfig,
     )

litellm/_lazy_imports_registry.py

@@ -171,6 +171,7 @@ LLM_CONFIG_NAMES: Final = (
     "DatabricksDecisionsConfig",
     "HostedVLLMDecisionsConfig",
     "LMStudioDecisionsConfig",
+    "OllamaDecisionsConfig",
     "LayaDecisionsConfig",
     "BespokeDecisionsConfig",
     "OpenAIDecisionsConfig",
@@ -749,6 +750,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "HostedVLLMDecisionsConfig",
     ),
     "LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"),
+    "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
     "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
     "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
     "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),

litellm/a2a_protocol/providers/bedrock_agentcore/transformation.py

@@ -19,6 +19,7 @@ from litellm.a2a_protocol.utils import (
 )
 from litellm.exceptions import BadRequestError
 from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
 
 RUNTIME_SESSION_ID_MIN_LENGTH: Final = 33
 RUNTIME_SESSION_ID_MAX_LENGTH: Final = 256
@@ -50,7 +51,7 @@ _RESERVED_PREFIX_HEADERS: Final[tuple[str, ...]] = (
 class _SSELineSource(Protocol):
     """Minimal streaming-response surface used to read SSE lines."""
 
-    def aiter_lines(self) -> AsyncIterator[str]: ...
+    def aiter_text(self) -> AsyncIterator[str]: ...
 
 
 def _filter_reserved_headers(
@@ -244,7 +245,7 @@ class BedrockAgentCoreA2ATransformation:
         Yields:
             Parsed JSON dicts from SSE data lines
         """
-        async for line in response.aiter_lines():
+        async for line in aiter_sse_lines(response.aiter_text()):
             line = line.strip()
             if not line:
                 continue

litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py

@@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     get_async_httpx_client,
 )
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
 from litellm.types.llms.custom_http import httpxSpecialProvider
 
 _IBM_CLOUD_IAM_URL: Final = "https://iam.cloud.ibm.com/identity/token"
@@ -74,7 +75,7 @@ class _WXORun(TypedDict, total=False):
 
 
 class _SSELineSource(Protocol):
-    def aiter_lines(self) -> AsyncIterator[str]: ...
+    def aiter_text(self) -> AsyncIterator[str]: ...
 
 
 class _WXOView(TypedDict, total=False):
@@ -231,7 +232,7 @@ class WatsonxOrchestrateHandler:
     async def _accumulate_wxo_sse_text(response: _SSELineSource) -> str:
         source: Final[_WXOView] = {"sse_source": response}
         accumulated_text = ""
-        async for line in source["sse_source"].aiter_lines():
+        async for line in aiter_sse_lines(source["sse_source"].aiter_text()):
             if not line.startswith("data:"):
                 continue
             data_str = line[5:].strip()

litellm/accounting/__init__.py

@@ -0,0 +1,62 @@
+"""Public accounting contracts. Importing these values does not enable accounting."""
+
+from litellm.accounting.measurements import measurements_for
+from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter
+from litellm.accounting.types import (
+    AdmissionDecision,
+    AggregateQuery,
+    AggregateRow,
+    CostEvent,
+    DeliveryState,
+    DeliveryTarget,
+    Dimension,
+    DimensionFilter,
+    DimensionName,
+    EventKind,
+    Granularity,
+    InboundUsageEvent,
+    Measurement,
+    Metric,
+    MetricName,
+    Multiplier,
+    NamedCost,
+    OutboundUsageEvent,
+    Outcome,
+    PricedQuantity,
+    PricingSnapshot,
+    Rate,
+    Unpriced,
+    validate_aggregate_rows,
+)
+
+__all__ = [
+    "AdmissionDecision",
+    "AggregateQuery",
+    "AggregateRow",
+    "BudgetEnforcer",
+    "CostAggregator",
+    "CostCompiler",
+    "CostEvent",
+    "DeliveryState",
+    "DeliveryTarget",
+    "Dimension",
+    "DimensionFilter",
+    "DimensionName",
+    "EventKind",
+    "Granularity",
+    "InboundUsageEvent",
+    "Measurement",
+    "Metric",
+    "MetricName",
+    "Multiplier",
+    "NamedCost",
+    "OutboundUsageEvent",
+    "Outcome",
+    "PricedQuantity",
+    "PricingSnapshot",
+    "Rate",
+    "Unpriced",
+    "UsageMeter",
+    "measurements_for",
+    "validate_aggregate_rows",
+]

litellm/accounting/measurements.py

@@ -0,0 +1,13 @@
+from typing import Final
+
+from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent
+
+
+def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]:
+    """Return the complete contribution set for one event, ready for one apply call."""
+    metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics
+    occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at
+    return tuple(
+        Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric)
+        for metric in metrics or ()
+    )

litellm/accounting/protocols.py

@@ -0,0 +1,85 @@
+"""Contracts only. Implementations own persistence, pricing, projections and live budgets.
+
+The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
+Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
+model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
+These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
+Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
+T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
+"""
+
+from typing import Protocol
+
+from litellm.accounting.types import (
+    AdmissionDecision,
+    AggregateQuery,
+    AggregateRow,
+    CostEvent,
+    Dimension,
+    InboundUsageEvent,
+    Measurement,
+    Metric,
+    OutboundUsageEvent,
+    Unpriced,
+)
+
+
+class UsageMeter(Protocol):
+    async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
+        """Acknowledge commit, no-op identical IDs and reject conflicting content.
+
+        Persist the event and its request-scoped pricing context before acknowledging.
+        HTTP acknowledgement is a separate policy. Unknown usage remains None.
+        """
+        ...
+
+
+class CostCompiler(Protocol):
+    async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
+        """Reuse the committed charge or finalize one charge and its delivery intent.
+
+        Resolve captured request-scoped context by event identity and cost-map rates
+        at compilation. Persist the resolved inputs in the required snapshot. Replay
+        reuses the committed result and never consumes free allowance again.
+        Missing pricing is Unpriced, never an inferred zero.
+        """
+        ...
+
+
+class CostAggregator(Protocol):
+    async def apply(self, measurements: tuple[Measurement, ...]) -> None:
+        """Apply one source event's complete contribution set per target/version.
+
+        Each target commits all contributions and its receipt atomically. A receipt
+        for the first metric must not suppress the rest. Usage need not wait for cost.
+        """
+        ...
+
+    async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
+        """Return disjoint [start, end) rows per group using existing daily projections.
+
+        Filters combine with AND, values within a filter with IN. DAY emits daily
+        buckets; None emits whole-interval totals. Reject unsupported metric/scope
+        combinations. Different groups may overlap, including non-additive tags.
+        """
+        ...
+
+
+class BudgetEnforcer(Protocol):
+    async def reserve(
+        self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...]
+    ) -> AdmissionDecision:
+        """Apply existing admission policies and estimates; duplicate IDs add no hold."""
+        ...
+
+    async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None:
+        """Settle the committed charge with atomic deduplication and hold mutation.
+
+        Preserve scope/window attribution and retry unfinished settlement after
+        restart. SQL projection receipts do not provide live-counter atomicity.
+        """
+        ...
+
+    async def release(self, reservation_id: str) -> None:
+        """Idempotently release unused exposure at the request boundary, preserving charges."""
+        ...

litellm/accounting/types.py

@@ -0,0 +1,428 @@
+"""Accounting values use UTC instants, decimal strings on the wire and USD charges."""
+
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from decimal import Decimal
+from enum import Enum
+from typing import Annotated, Final, Literal, TypeAlias
+
+from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator
+from typing_extensions import Self
+
+from litellm.types.utils import CallTypes
+
+_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True)
+Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")]
+Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)]
+
+
+def _utc(value: datetime) -> datetime:
+    return value.astimezone(timezone.utc)
+
+
+Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)]
+
+
+class MetricName(str, Enum):
+    """Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens."""
+
+    REQUESTS = "requests"
+    SUCCESSFUL_REQUESTS = "successful_requests"
+    FAILED_REQUESTS = "failed_requests"
+    PROMPT_TOKENS = "prompt_tokens"
+    COMPLETION_TOKENS = "completion_tokens"
+    REASONING_TOKENS = "reasoning_tokens"
+    TOTAL_TOKENS = "total_tokens"
+    CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens"
+    CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens"
+    CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr"
+    SPEND = "spend"
+
+
+class DimensionName(str, Enum):
+    KEY = "key"
+    USER = "user"
+    TEAM = "team"
+    ORGANIZATION = "organization"
+    END_USER = "end_user"
+    MODEL = "model"
+    MODEL_GROUP = "model_group"
+    DEPLOYMENT = "deployment"
+    PROVIDER = "provider"
+    TOOL = "tool"
+    AGENT = "agent"
+    PROJECT = "project"
+    MODEL_ACCESS_GROUP = "model_access_group"
+    API_BASE = "api_base"
+    SERVICE_TIER = "service_tier"
+    TAG = "tag"
+
+
+class Outcome(str, Enum):
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+class Granularity(str, Enum):
+    DAY = "day"
+
+
+class EventKind(str, Enum):
+    INBOUND_USAGE = "inbound_usage"
+    OUTBOUND_USAGE = "outbound_usage"
+    COST = "cost"
+
+
+class DeliveryState(str, Enum):
+    PENDING = "pending"
+    APPLIED = "applied"
+    PARKED = "parked"
+
+
+class DeliveryTarget(str, Enum):
+    """One member per existing table the projector writes; each is applied and receipted independently."""
+
+    KEY_SPEND = "LiteLLM_VerificationToken"
+    USER_SPEND = "LiteLLM_UserTable"
+    TEAM_SPEND = "LiteLLM_TeamTable"
+    TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership"
+    ORGANIZATION_SPEND = "LiteLLM_OrganizationTable"
+    ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership"
+    END_USER_SPEND = "LiteLLM_EndUserTable"
+    AGENT_SPEND = "LiteLLM_AgentsTable"
+    TAG_SPEND = "LiteLLM_TagTable"
+    PROJECT_SPEND = "LiteLLM_ProjectTable"
+    MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable"
+    BUDGET_SPEND = "LiteLLM_BudgetTable"
+    BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend"
+    DAILY_USER_SPEND = "LiteLLM_DailyUserSpend"
+    DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend"
+    DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend"
+    DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend"
+    DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend"
+    DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend"
+    DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage"
+    DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend"
+    AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend"
+
+
+_SUBSET_PAIRS: Final = (
+    (MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS),
+    (MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS),
+)
+
+
+def _check_subsets(values: Mapping[str, Decimal]) -> None:
+    for part, whole in _SUBSET_PAIRS:
+        if part.value in values and whole.value in values and values[part.value] > values[whole.value]:
+            raise ValueError(f"{part.value} cannot exceed {whole.value}")
+
+
+class Metric(BaseModel):
+    model_config = _CONFIG
+
+    name: MetricName
+    value: Quantity
+    unit: Literal["count", "USD"] = "count"
+
+    @model_validator(mode="after")
+    def validate_unit(self) -> Self:
+        if (self.name is MetricName.SPEND) != (self.unit == "USD"):
+            raise ValueError("SPEND requires USD; usage metrics require count")
+        if self.unit == "count" and self.value != self.value.to_integral_value():
+            raise ValueError("Count metrics require integral values")
+        return self
+
+
+class Dimension(BaseModel):
+    model_config = _CONFIG
+
+    name: DimensionName
+    value: Identifier
+
+
+_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP})
+
+
+def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]:
+    if len(frozenset(values)) != len(values):
+        raise ValueError("Duplicate dimension values")
+    single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS)
+    if len(frozenset(single_names)) != len(single_names):
+        raise ValueError("Only tag and model access group dimensions may have multiple values")
+    return values
+
+
+Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)]
+_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS))
+
+
+def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]:
+    if len(frozenset(value.name for value in values)) != len(values):
+        raise ValueError("Duplicate metric names")
+    return values
+
+
+Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)]
+
+
+def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None:
+    if end < start or (end == start and not allow_empty):
+        raise ValueError("End must follow start")
+
+
+class InboundUsageEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    request_id: Identifier
+    call_type: CallTypes
+    started_at: Instant
+    ended_at: Instant
+    outcome: Outcome
+    dimensions: Dimensions
+    metrics: Metrics
+
+    @model_validator(mode="after")
+    def validate_caller_activity(self) -> Self:
+        _interval(self.started_at, self.ended_at, allow_empty=True)
+        if any(metric.name not in _CALLER_METRICS for metric in self.metrics):
+            raise ValueError("Inbound events own only caller request metrics")
+        if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics):
+            raise ValueError("Inbound events count exactly one caller request")
+        for metric in self.metrics:
+            if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED):
+                raise ValueError("Successful request count must match outcome")
+            if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED):
+                raise ValueError("Failed request count must match outcome")
+        return self
+
+
+class OutboundUsageEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    attempt_id: Identifier
+    request_id: Identifier | None
+    retry_of_attempt_id: Identifier | None
+    call_type: CallTypes
+    started_at: Instant
+    ended_at: Instant
+    outcome: Outcome
+    dimensions: Dimensions
+    metrics: Metrics | None
+
+    @model_validator(mode="after")
+    def validate_attempt_usage(self) -> Self:
+        _interval(self.started_at, self.ended_at, allow_empty=True)
+        if self.retry_of_attempt_id == self.attempt_id:
+            raise ValueError("An attempt cannot retry itself")
+        if self.metrics is not None and any(
+            metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics
+        ):
+            raise ValueError("Outbound events own only attempt usage metrics")
+        if self.metrics is not None:
+            _check_subsets({metric.name.value: metric.value for metric in self.metrics})
+        return self
+
+
+PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"]
+
+
+class Rate(BaseModel):
+    model_config = _CONFIG
+
+    measure: Identifier
+    unit: PricingUnit
+    usd_per_unit: Quantity
+
+
+class PricedQuantity(BaseModel):
+    """A pricing input in its billing unit. Only seconds may be fractional."""
+
+    model_config = _CONFIG
+
+    measure: Identifier
+    unit: PricingUnit
+    value: Quantity
+
+    @model_validator(mode="after")
+    def validate_discrete_quantity(self) -> Self:
+        if self.unit != "second" and self.value != self.value.to_integral_value():
+            raise ValueError("Discrete priced quantities require integral values")
+        return self
+
+
+class Multiplier(BaseModel):
+    """A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5."""
+
+    model_config = _CONFIG
+
+    name: Identifier
+    value: Identifier
+    factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)]
+
+
+class NamedCost(BaseModel):
+    """A free-form extra charge by name, e.g. azure_model_router_flat_cost."""
+
+    model_config = _CONFIG
+
+    name: Identifier
+    amount: Quantity
+
+
+class PricingSnapshot(BaseModel):
+    """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
+
+    Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
+    resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
+    Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
+    Component sums and float tolerances belong to reconciliation, not this structural contract.
+    """
+
+    model_config = _CONFIG
+
+    schema_version: Literal[1] = 1
+    calculator_version: Identifier
+    model: Identifier
+    provider: Identifier | None
+    pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"]
+    priced_model_key: Identifier
+    multipliers: tuple[Multiplier, ...] = ()
+    rates: tuple[Rate, ...] = ()
+    usage: tuple[PricedQuantity, ...] = ()
+    input_cost: Quantity | None = None
+    output_cost: Quantity | None = None
+    cache_read_cost: Quantity | None = None
+    cache_creation_cost: Quantity | None = None
+    reasoning_cost: Quantity | None = None
+    tool_usage_cost: Quantity | None = None
+    guardrail_cost: Quantity | None = None
+    additional_costs: tuple[NamedCost, ...] = ()
+    original_cost: Quantity | None = None
+    discount_percent: Quantity | None = None
+    discount_amount: Quantity | None = None
+    margin_percent: Quantity | None = None
+    margin_fixed_amount: Quantity | None = None
+    margin_total_amount: Quantity | None = None
+    upstream_reported_cost: Quantity | None = None
+    total_cost: Quantity
+
+    @model_validator(mode="after")
+    def validate_snapshot(self) -> Self:
+        rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates)
+        quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage)
+        if len(rate_keys) != len(self.rates):
+            raise ValueError("Duplicate rate measures and units")
+        if len(quantity_keys) != len(self.usage):
+            raise ValueError("Duplicate priced quantity measures and units")
+        if not rate_keys.issubset(quantity_keys):
+            raise ValueError("Every rate requires one priced quantity with the same measure and unit")
+        if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs):
+            raise ValueError("Duplicate additional cost names")
+        if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers):
+            raise ValueError("Duplicate multiplier names")
+        _check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"})
+        return self
+
+
+class CostEvent(BaseModel):
+    model_config = _CONFIG
+
+    event_id: Identifier
+    usage_event_id: Identifier
+    occurred_at: Instant
+    dimensions: Dimensions
+    charges: Metrics
+    snapshot: PricingSnapshot
+
+    @model_validator(mode="after")
+    def validate_charge(self) -> Self:
+        if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND:
+            raise ValueError("A cost event requires one authoritative USD spend metric")
+        if self.event_id == self.usage_event_id:
+            raise ValueError("Cost and usage events require distinct identities")
+        if self.snapshot.total_cost != self.charges[0].value:
+            raise ValueError("Snapshot total must equal the authoritative spend")
+        return self
+
+
+class Unpriced(BaseModel):
+    model_config = _CONFIG
+
+    usage_event_id: Identifier
+    reason: Identifier
+
+
+class Measurement(BaseModel):
+    model_config = _CONFIG
+
+    source_event_id: Identifier
+    occurred_at: Instant
+    dimensions: Dimensions
+    metric: Metric
+
+
+class DimensionFilter(BaseModel):
+    model_config = _CONFIG
+
+    name: DimensionName
+    values: Annotated[tuple[Identifier, ...], Field(min_length=1)]
+
+
+class AggregateQuery(BaseModel):
+    model_config = _CONFIG
+
+    metric: MetricName
+    filters: tuple[DimensionFilter, ...]
+    group_by: tuple[DimensionName, ...]
+    start: Instant
+    end: Instant
+    granularity: Granularity | None
+
+    @model_validator(mode="after")
+    def validate_query(self) -> Self:
+        _interval(self.start, self.end)
+        if len(frozenset(item.name for item in self.filters)) != len(self.filters):
+            raise ValueError("Use one IN filter per dimension; filters combine with AND")
+        if len(frozenset(self.group_by)) != len(self.group_by):
+            raise ValueError("Duplicate group-by dimensions")
+        if any(
+            (instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0)
+            for instant in (self.start, self.end)
+        ):
+            raise ValueError("Initial reporting queries require UTC day boundaries")
+        return self
+
+
+class AggregateRow(BaseModel):
+    model_config = _CONFIG
+
+    start: Instant
+    end: Instant
+    dimensions: Dimensions
+    value: Quantity
+
+    @model_validator(mode="after")
+    def validate_bounds(self) -> Self:
+        _interval(self.start, self.end)
+        return self
+
+
+def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]:
+    """Validate [start, end) buckets per group; overlapping tag groups are not additive."""
+    groups: Final = frozenset(frozenset(row.dimensions) for row in rows)
+    for group in groups:
+        ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start)
+        if any(left.end > right.start for left, right in zip(ordered, ordered[1:])):
+            raise ValueError("Aggregate buckets overlap within a dimension group")
+    return rows
+
+
+class AdmissionDecision(BaseModel):
+    model_config = _CONFIG
+
+    allowed: bool
+    reason: Identifier | None = None

litellm/caching/in_memory_cache.py

@@ -274,7 +274,9 @@ class InMemoryCache(BaseCache):
     ) -> list[float] | None:
         results: Final = []
         for increment in increment_list:
-            result = await self.async_increment(increment["key"], increment["increment_value"], **kwargs)
+            result = await self.async_increment(
+                increment["key"], increment["increment_value"], ttl=increment["ttl"], **kwargs
+            )
             results.append(result)
         return results
 

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -3,7 +3,6 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 """
 
 import json
-import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
 from itertools import accumulate, chain
 from types import MappingProxyType
@@ -35,6 +34,7 @@ from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.bridges.completion_transformation import (
     CompletionTransformationBridge,
 )
+from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool
 from litellm.responses.sse_output_recovery import (
     parse_sse_json_chunk,
@@ -43,8 +43,8 @@ from litellm.responses.sse_output_recovery import (
 )
 from litellm.responses.utils import (
     ResponsesAPIRequestUtils,
-    auto_reasoning_summary,
     normalize_responses_api_stream_options,
+    opt_in_auto_reasoning_summary,
 )
 from litellm.types.llms.openai import (
     ChatCompletionAnnotation,
@@ -1352,7 +1352,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         recovered_output_items: Final[dict[int, dict[str, object]]] = {}
         recovered_text_only_items: Final[dict[int, dict[str, object]]] = {}
 
-        for chunk in raw_sse.splitlines():
+        for chunk in split_sse_lines(raw_sse):
             parsed_chunk = parse_sse_json_chunk(chunk)
             if parsed_chunk is None:
                 continue
@@ -1745,19 +1745,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 **cast(Reasoning, reasoning_effort)  # cast-ok: dict is forwarded verbatim to the provider
             )
 
-        # Check if auto-summary is enabled via flag or environment variable
-        # Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var
-        auto_summary_enabled: Final = (
-            litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
-        )
-
+        summary: Final = opt_in_auto_reasoning_summary(model, custom_llm_provider)
         return (
-            Reasoning(
-                effort=reasoning_effort,
-                summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
-            )
-            if auto_summary_enabled
-            else Reasoning(effort=reasoning_effort)
+            Reasoning(effort=reasoning_effort)
+            if summary is None
+            else Reasoning(effort=reasoning_effort, summary=summary)
         )
 
     _map_reasoning_effort = map_reasoning_effort

litellm/constants.py

@@ -130,6 +130,7 @@ MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
 S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
 S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
 MAX_FILE_LIST_LIMIT: Final = 10000
+ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
 GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10
@@ -764,7 +765,6 @@ ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
 # Used for web search interception across providers
 LITELLM_WEB_SEARCH_TOOL_NAME: Final = "litellm_web_search"
 
-DEFAULT_IMAGE_ENDPOINT_MODEL: Final = "dall-e-2"
 DEFAULT_VIDEO_ENDPOINT_MODEL: Final = "sora-2"
 
 DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = get_env_int("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)
@@ -1097,6 +1097,7 @@ openai_compatible_providers: Final[list[str]] = [
     "llamafile",
     "lm_studio",
     "galadriel",
+    "zerogpu",
     "github_copilot",  # GitHub Copilot Chat API
     "chatgpt",  # ChatGPT subscription API
     "novita",

litellm/cost_calculator.py

@@ -1592,14 +1592,6 @@ def completion_cost(
                     for part in ws_tier_parts
                 )
 
-        if (
-            (call_type == "aimage_generation" or call_type == "image_generation")
-            and model is not None
-            and isinstance(model, str)
-            and len(model) == 0
-            and custom_llm_provider == "azure"
-        ):
-            model = "dall-e-2"  # for dall-e-2, azure expects an empty model name
         # Handle Inputs to completion_cost
         prompt_tokens = 0
         prompt_characters: int | None = None
@@ -1674,7 +1666,8 @@ def completion_cost(
             raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
         )
 
-        for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
+        for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names):
+            model = candidate_model
             try:
                 if verbose_logger.isEnabledFor(logging.DEBUG):
                     verbose_logger.debug("selected model name for cost calculation: %s", model)

litellm/google_genai/streaming_iterator.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
 
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
 )
@@ -131,9 +132,9 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent
         self.model = model
         self.generate_content_provider_config = generate_content_provider_config
         self.litellm_metadata = litellm_metadata
-        # Gemini streamGenerateContent uses SSE line framing; iter_lines keeps
+        # Gemini streamGenerateContent uses SSE line framing; iter_sse_lines keeps
         # large inlineData payloads (e.g. image/jpeg) intact within one event.
-        self.stream_iterator = response.iter_lines()
+        self.stream_iterator = iter_sse_lines(response.iter_text())
 
     def __iter__(self):
         return self
@@ -182,9 +183,9 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
         self.model = model
         self.generate_content_provider_config = generate_content_provider_config
         self.litellm_metadata = litellm_metadata
-        # Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps
+        # Gemini streamGenerateContent uses SSE line framing; aiter_sse_lines keeps
         # large inlineData payloads (e.g. image/jpeg) intact within one event.
-        self.stream_iterator = response.aiter_lines()
+        self.stream_iterator = aiter_sse_lines(response.aiter_text())
 
     def __aiter__(self):
         return self

litellm/images/main.py

@@ -1,9 +1,9 @@
 import asyncio
 import contextvars
 import importlib
-from collections.abc import Coroutine, Mapping
+from collections.abc import Collection, Coroutine, Mapping
 from functools import partial
-from typing import TYPE_CHECKING, Final, Literal, cast, overload
+from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
 
 from pydantic import TypeAdapter, ValidationError
 
@@ -16,7 +16,6 @@ import litellm
 
 # client is imported from litellm as it's a decorator
 from litellm import client
-from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL
 from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
 from litellm.exceptions import LiteLLMUnknownProvider
 from litellm.litellm_core_utils.get_llm_provider_logic import (
@@ -61,6 +60,7 @@ from litellm.types.utils import (
     ProviderRoutingHints,
 )
 from litellm.utils import (
+    EXTRA_BODY_ROUTING_KEYS,
     ImageResponse,
     ProviderConfigManager,
     get_llm_provider,
@@ -298,17 +298,14 @@ def image_generation(
             if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/"))
             else None
         )
-        dynamic_api_key: str | None = None
-        if model is not None or custom_llm_provider is not None:
-            model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
-                model=model,
-                custom_llm_provider=custom_llm_provider,
-                api_base=api_base,
-                litellm_params=chatgpt_params,
-            )
-        else:
-            model = "dall-e-2"
-            custom_llm_provider = "openai"  # default to dall-e-2 on openai
+        if model is None and custom_llm_provider is None:
+            raise litellm.BadRequestError(message="model is required", model="", llm_provider="")
+        model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            api_base=api_base,
+            litellm_params=chatgpt_params,
+        )
         model_response.hidden_params["model"] = model
         openai_params: Final = [
             "user",
@@ -466,7 +463,7 @@ def image_generation(
             litellm.LlmProviders.QWENCLOUD,
             litellm.LlmProviders.QWEN_AI_PLATFORM,
             litellm.LlmProviders.EDENAI,
-        ):
+        ) or (custom_llm_provider == litellm.LlmProviders.TENCENT and image_generation_config is not None):
             if image_generation_config is None:
                 raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
 
@@ -711,7 +708,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
 @client
 def image_variation(
     image: FileTypes,
-    model: str = "dall-e-2",  # set to dall-e-2 by default - like OpenAI.
+    model: str | None = None,
     n: int = 1,
     response_format: Literal["url", "b64_json"] = "url",
     size: str | None = None,
@@ -727,7 +724,7 @@ def image_variation(
     litellm_params: Final = get_litellm_params(**kwargs)
     # get the custom llm provider
     model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
-        model=model,
+        model=model or "",
         custom_llm_provider=litellm_params.get("custom_llm_provider", None),
         api_base=litellm_params.get("api_base", None),
         api_key=litellm_params.get("api_key", None),
@@ -746,7 +743,7 @@ def image_variation(
     response: ImageResponse | None = None
 
     provider_config: Final = ProviderConfigManager.get_provider_model_info(
-        model=model or "",  # openai defaults to dall-e-2
+        model=model,
         provider=llm_provider,
     )
 
@@ -771,7 +768,7 @@ def image_variation(
             model_response=model_response,
             api_key=api_key,
             api_base=api_base,
-            model=model,
+            model=model or None,
             image=image,
             timeout=litellm_params.get("timeout", None),
             custom_llm_provider=custom_llm_provider,
@@ -789,7 +786,7 @@ def image_variation(
             model_response=model_response,
             api_key=api_key,
             api_base=api_base,
-            model=model,
+            model=model or None,
             image=image,
             timeout=litellm_params.get("timeout", None) or DEFAULT_REQUEST_TIMEOUT,
             custom_llm_provider=custom_llm_provider,
@@ -807,6 +804,32 @@ def image_variation(
     return response
 
 
+_IMAGE_EDIT_OPENAI_PARAMS: Final = frozenset(get_type_hints(ImageEditOptionalRequestParams))
+_DROP_PARAM_KWARGS: Final = frozenset({"drop_params", "additional_drop_params"})
+_ADDITIONAL_DROP_PARAMS: Final = TypeAdapter[tuple[str, ...]](tuple[str, ...])
+
+
+def _listed_drop_params(additional_drop_params: object) -> tuple[str, ...]:
+    try:
+        return _ADDITIONAL_DROP_PARAMS.validate_python(additional_drop_params)
+    except ValidationError:
+        return ()
+
+
+def _tokenhub_passthrough_edit_params(
+    non_default_params: Mapping[str, object],
+    extra_body: Mapping[str, object] | None,
+    mapped_params: Collection[str],
+    additional_drop_params: Collection[str] | None,
+) -> Mapping[str, object]:
+    dropped: Final = frozenset(additional_drop_params or ())
+    unmapped_excluded: Final = _DROP_PARAM_KWARGS | _IMAGE_EDIT_OPENAI_PARAMS | frozenset(mapped_params) | dropped
+    return {
+        **{key: value for key, value in non_default_params.items() if key not in unmapped_excluded},
+        **{key: value for key, value in (extra_body or {}).items() if key not in dropped | EXTRA_BODY_ROUTING_KEYS},
+    }
+
+
 @client
 def image_edit(
     image: FileTypes | list[FileTypes] | None = None,
@@ -874,7 +897,7 @@ def image_edit(
         # get llm provider logic
         litellm_params: Final = GenericLiteLLMParams(**kwargs)
         model, custom_llm_provider, _, _ = get_llm_provider(
-            model=model or DEFAULT_IMAGE_ENDPOINT_MODEL,
+            model=model or "",
             custom_llm_provider=custom_llm_provider,
         )
 
@@ -935,22 +958,24 @@ def image_edit(
             raise ValueError(f"image edit is not supported for {custom_llm_provider}")
 
         local_vars.update(kwargs)
+        provider_supported_params: Final[frozenset[str]] = frozenset(
+            image_edit_provider_config.get_supported_openai_params(model)
+        )
         # Get ImageEditOptionalRequestParams with only valid parameters
         image_edit_optional_params: Final[ImageEditOptionalRequestParams] = (
             _get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
                 local_vars,
-                provider_supported_params=frozenset(
-                    image_edit_provider_config.get_supported_openai_params(model)
-                ).intersection(non_default_params),
+                provider_supported_params=provider_supported_params.intersection(non_default_params),
             )
         )
         # Get optional parameters for the responses API
+        additional_drop_params: Final = _listed_drop_params(kwargs.get("additional_drop_params"))
         image_edit_request_params: Final[dict] = _get_ImageEditRequestUtils().get_optional_params_image_edit(
             model=model,
             image_edit_provider_config=image_edit_provider_config,
             image_edit_optional_params=image_edit_optional_params,
             drop_params=kwargs.get("drop_params"),
-            additional_drop_params=kwargs.get("additional_drop_params"),
+            additional_drop_params=additional_drop_params,
         )
 
         if image_edit_provider_config.use_multipart_form_data() and (
@@ -1039,7 +1064,19 @@ def image_edit(
             image=images,
             prompt=prompt,
             image_edit_provider_config=image_edit_provider_config,
-            image_edit_optional_request_params=image_edit_request_params,
+            image_edit_optional_request_params={
+                **image_edit_request_params,
+                **(
+                    _tokenhub_passthrough_edit_params(
+                        non_default_params,
+                        extra_body if isinstance(extra_body, dict) else None,
+                        provider_supported_params,
+                        additional_drop_params,
+                    )
+                    if custom_llm_provider == litellm.LlmProviders.TENCENT.value
+                    else {}
+                ),
+            },
             custom_llm_provider=custom_llm_provider,
             litellm_params=litellm_params,
             logging_obj=litellm_logging_obj,

litellm/images/utils.py

@@ -1,6 +1,9 @@
+import base64
+import os
 from collections.abc import Collection, Mapping
 from io import BufferedReader, BytesIO
-from typing import Any, Final, cast, get_type_hints
+from pathlib import Path
+from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
 
 import litellm
 from litellm.litellm_core_utils.token_counter import get_image_type
@@ -16,7 +19,7 @@ class ImageEditRequestUtils:
         image_edit_provider_config: BaseImageEditConfig,
         image_edit_optional_params: ImageEditOptionalRequestParams,
         drop_params: bool | None = None,
-        additional_drop_params: list[str] | None = None,
+        additional_drop_params: Collection[str] | None = None,
     ) -> dict:
         """
         Get optional parameters for the image edit API.
@@ -145,3 +148,36 @@ class ImageEditRequestUtils:
         except Exception:
             # If anything goes wrong, default to PNG
             return FILE_MIME_TYPES[FileType.PNG]
+
+
+@runtime_checkable
+class _SeekableBinaryReader(Protocol):
+    def tell(self) -> int: ...
+
+    def seek(self, offset: int) -> int: ...
+
+    def read(self) -> bytes: ...
+
+
+def _read_image_bytes(image: object) -> bytes:
+    if isinstance(image, bytes):
+        return image
+    if isinstance(image, tuple):
+        return _read_image_bytes(image[1])
+    if isinstance(image, os.PathLike):
+        return Path(image).read_bytes()
+    if isinstance(image, _SeekableBinaryReader):
+        position: Final = image.tell()
+        image.seek(0)
+        data: Final = image.read()
+        image.seek(position)
+        return data
+    raise ValueError(f"Unsupported image type for image edit: {type(image).__name__}")
+
+
+def image_to_data_url(image: object) -> str:
+    if isinstance(image, str):
+        return image
+    image_bytes: Final = _read_image_bytes(image)
+    mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
+    return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"

litellm/integrations/otel/model/spans.py

@@ -338,6 +338,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_ToolTable",
         "LiteLLM_TelemetryReport",
         "LiteLLM_AccessGroupTable",
+        "LiteLLM_AccountingDelivery",
+        "LiteLLM_AccountingEvent",
         "LiteLLM_ClaudeCodePluginTable",
         "LiteLLM_MemoryTable",
         "LiteLLM_AdaptiveRouterState",
@@ -434,6 +436,7 @@ _POSTGRES_OPERATION_BY_CALL_TYPE: Final[Mapping[str, PostgresOperation]] = Mappi
         "reset_budget_windows": PostgresOperation("select", None),
         "write_budget_windows": PostgresOperation("update", None),
         "roll_window_spend_row": PostgresOperation("update", None),
+        "open_window_spend_row": PostgresOperation("upsert", None),
         "seed_window_spend": PostgresOperation("select", None),
         "select_window_spend_rows": PostgresOperation("select", None),
         "commit_window_spend_updates": PostgresOperation("upsert", None),

litellm/integrations/websearch_interception/handler.py

@@ -97,6 +97,8 @@ WEBSEARCH_NATIVE_BLOCKS_METADATA_KEY: Final = "websearch_native_blocks"
 
 _RESPONSE_CONTENT_FIELD: Final = "content"
 
+_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX: Final = "Perform a web search for the query:"
+
 _ResponseT: Final = TypeVar("_ResponseT")
 
 
@@ -363,7 +365,8 @@ class WebSearchInterceptionLogger(CustomLogger):
             get_last_user_message,
         )
 
-        query: Final = get_last_user_message(cast(list[AllMessageValues], messages))
+        last_user_message: Final = get_last_user_message(cast(list[AllMessageValues], messages)) or ""
+        query: Final = last_user_message.removeprefix(_CLAUDE_CODE_WEB_SEARCH_PROMPT_PREFIX).strip()
         if not query:
             return None
 

litellm/interactions/__init__.py

@@ -32,11 +32,11 @@ Usage:
 
     # Create a managed agent on the provider side
     result = litellm.interactions.agents.create(
-        name="waverunner",
+        id="waverunner",
         custom_llm_provider="gemini",
         api_key="...",
-        base_agent="gemini-2.5-flash",
-        instructions="You are a helpful assistant.",
+        base_agent="antigravity-preview-09-2026",
+        system_instruction="You are a helpful assistant.",
     )
 
 Methods:

litellm/interactions/agents/__init__.py

@@ -3,11 +3,11 @@ litellm.interactions.agents
 
 Full CRUD SDK for provider-side managed agents (e.g. Gemini v1beta/agents).
 
-    litellm.interactions.agents.create(name=..., ...)
+    litellm.interactions.agents.create(id=..., ...)
     litellm.interactions.agents.list(api_key=...)
-    litellm.interactions.agents.get(name=..., ...)
-    litellm.interactions.agents.delete(name=..., ...)
-    litellm.interactions.agents.list_versions(name=..., ...)
+    litellm.interactions.agents.get(id=..., ...)
+    litellm.interactions.agents.delete(id=..., ...)
+    litellm.interactions.agents.list_versions(id=..., ...)
 
 Async counterparts: acreate, alist, aget, adelete, alist_versions
 """

litellm/interactions/agents/http_handler.py

@@ -17,6 +17,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.types.agents import (
+    AgentCreateRequest,
     AgentCreateResponse,
     AgentDeleteResult,
     AgentListResponse,
@@ -35,7 +36,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
     def create_agent(
         self,
         agents_api_config: BaseAgentsAPIConfig,
-        name: str,
+        create_request: AgentCreateRequest,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
         extra_headers: dict[str, str] | None = None,
@@ -47,7 +48,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
         if _is_async:
             return self.async_create_agent(
                 agents_api_config=agents_api_config,
-                name=name,
+                create_request=create_request,
                 litellm_params=litellm_params,
                 logging_obj=logging_obj,
                 extra_headers=extra_headers,
@@ -63,12 +64,13 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
             api_base=litellm_params.get("api_base"),
             litellm_params=dict(litellm_params),
         )
-        data: Final = agents_api_config.transform_create_request(name=name, litellm_params=dict(litellm_params))
-        if extra_body:
-            data.update(extra_body)
+        data: Final = {
+            **agents_api_config.transform_create_request(create_request),
+            **(extra_body or {}),
+        }
 
         logging_obj.pre_call(
-            input=name,
+            input=create_request.id,
             api_key="",
             additional_args={
                 "complete_input_dict": data,
@@ -85,12 +87,12 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
             original_response=response.text,
             additional_args={"complete_input_dict": data},
         )
-        return agents_api_config.transform_create_response(raw_response=response, name=name)
+        return agents_api_config.transform_create_response(raw_response=response, name=create_request.id)
 
     async def async_create_agent(
         self,
         agents_api_config: BaseAgentsAPIConfig,
-        name: str,
+        create_request: AgentCreateRequest,
         litellm_params: GenericLiteLLMParams,
         logging_obj: LiteLLMLoggingObj,
         extra_headers: dict[str, str] | None = None,
@@ -106,12 +108,13 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
             api_base=litellm_params.get("api_base"),
             litellm_params=dict(litellm_params),
         )
-        data: Final = agents_api_config.transform_create_request(name=name, litellm_params=dict(litellm_params))
-        if extra_body:
-            data.update(extra_body)
+        data: Final = {
+            **agents_api_config.transform_create_request(create_request),
+            **(extra_body or {}),
+        }
 
         logging_obj.pre_call(
-            input=name,
+            input=create_request.id,
             api_key="",
             additional_args={
                 "complete_input_dict": data,
@@ -130,7 +133,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
             original_response=response.text,
             additional_args={"complete_input_dict": data},
         )
-        return agents_api_config.transform_create_response(raw_response=response, name=name)
+        return agents_api_config.transform_create_response(raw_response=response, name=create_request.id)
 
     # ------------------------------------------------------------------ #
     # LIST                                                                 #

litellm/interactions/agents/main.py

@@ -6,41 +6,47 @@ Usage:
 
     # Create
     response = litellm.interactions.agents.create(
-        name="waverunner",
+        id="waverunner",
         custom_llm_provider="gemini",
         api_key="...",
-        base_agent="gemini-2.5-flash",
-        instructions="You are a helpful assistant.",
+        base_agent="antigravity-preview-09-2026",
+        system_instruction="You are a helpful assistant.",
+        agent_config={"type": "antigravity", "model": "gemini-3.8-flash"},
     )
 
     # List
     response = litellm.interactions.agents.list(api_key="...", custom_llm_provider="gemini")
 
     # Get
-    response = litellm.interactions.agents.get(name="waverunner", api_key="...")
+    response = litellm.interactions.agents.get(id="waverunner", api_key="...")
 
     # Delete
-    result = litellm.interactions.agents.delete(name="waverunner", api_key="...")
+    result = litellm.interactions.agents.delete(id="waverunner", api_key="...")
 
     # List versions
-    result = litellm.interactions.agents.list_versions(name="waverunner", api_key="...")
+    result = litellm.interactions.agents.list_versions(id="waverunner", api_key="...")
 
     # Async versions: acreate, alist, aget, adelete, alist_versions
 """
 
 import asyncio
 import contextvars
-from collections.abc import Coroutine, Mapping
+from collections.abc import Coroutine, Mapping, Sequence
+from dataclasses import dataclass
 from functools import partial
+from types import MappingProxyType
 from typing import Any, Final
 
 import httpx
+from pydantic import ValidationError
 
 import litellm
 from litellm.interactions.agents.http_handler import agents_http_handler
 from litellm.interactions.agents.utils import get_provider_agents_api_config
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.types.agents import (
+    AgentCreateRequest,
     AgentCreateResponse,
     AgentDeleteResult,
     AgentListResponse,
@@ -89,14 +95,120 @@ def _make_logging_obj(
     return litellm_logging_obj
 
 
+@dataclass(frozen=True, slots=True)
+class _InvalidAgentRequest:
+    message: str
+
+
+def _alias_conflict(field: str, value: str | None, alias: str, alias_value: str | None) -> str | None:
+    if value is None or alias_value is None or value == alias_value:
+        return None
+    return f"'{alias}' is an alias of '{field}', but the request gives them different values. Pass only '{field}'."
+
+
+def _agent_id_or_error(id: str | None, name: str | None) -> str | _InvalidAgentRequest:
+    conflict: Final = _alias_conflict("id", id, "name", name)
+    if conflict is not None:
+        return _InvalidAgentRequest(conflict)
+    agent_id: Final = id if id is not None else name
+    if agent_id is None:
+        return _InvalidAgentRequest("Missing required parameter: 'id' (or its alias 'name').")
+    return agent_id
+
+
+def _required_agent_id(id: str | None, name: str | None, llm_provider: str) -> str:
+    agent_id: Final = _agent_id_or_error(id, name)
+    if isinstance(agent_id, _InvalidAgentRequest):
+        raise litellm.BadRequestError(message=agent_id.message, model=id or name or "", llm_provider=llm_provider)
+    return agent_id
+
+
 # ================================================================== #
 # CREATE                                                               #
 # ================================================================== #
 
 
+_CREATE_FIELD_SHAPES: Final = MappingProxyType(
+    {
+        "id": "a string",
+        "base_agent": "a string",
+        "agent_config": "an object",
+        "system_instruction": "a string",
+        "base_environment": "an environment id string or an object",
+        "description": "a string",
+        "tools": "a list of objects",
+    }
+)
+
+
+def _invalid_create_fields_message(invalid_fields: frozenset[str], caller_names: Mapping[str, str]) -> str:
+    problems: Final = "; ".join(
+        f"'{caller_names.get(field, field)}' must be {_CREATE_FIELD_SHAPES[field]}" for field in sorted(invalid_fields)
+    )
+    drop_hint: Final = "" if "id" in invalid_fields else " Set drop_params to drop invalid optional fields."
+    return f"Invalid agent create request: {problems}.{drop_hint}"
+
+
+def _validated_create_request(
+    fields: Mapping[str, object], drop_invalid_fields: bool, caller_names: Mapping[str, str]
+) -> AgentCreateRequest | _InvalidAgentRequest:
+    try:
+        return AgentCreateRequest.model_validate(fields)
+    except ValidationError as e:
+        invalid_fields: Final = frozenset(str(error["loc"][0]) for error in e.errors())
+        if drop_invalid_fields and "id" not in invalid_fields:
+            return AgentCreateRequest.model_validate(
+                {field: value for field, value in fields.items() if field not in invalid_fields}
+            )
+        return _InvalidAgentRequest(_invalid_create_fields_message(invalid_fields, caller_names))
+
+
+def _create_request_or_error(
+    id: str | None,
+    name: str | None,
+    base_agent: str | None,
+    agent_config: Mapping[str, object] | None,
+    system_instruction: str | None,
+    instructions: str | None,
+    base_environment: InteractionEnvironment | None,
+    description: str | None,
+    tools: Sequence[Mapping[str, object]] | None,
+    drop_invalid_fields: bool,
+) -> AgentCreateRequest | _InvalidAgentRequest:
+    agent_id: Final = _agent_id_or_error(id, name)
+    if isinstance(agent_id, _InvalidAgentRequest):
+        return agent_id
+    instruction_conflict: Final = _alias_conflict(
+        "system_instruction", system_instruction, "instructions", instructions
+    )
+    if instruction_conflict is not None and not drop_invalid_fields:
+        return _InvalidAgentRequest(instruction_conflict)
+    sent_only_instructions: Final = system_instruction is None and instructions is not None
+    return _validated_create_request(
+        MappingProxyType(
+            {
+                "id": agent_id,
+                "base_agent": base_agent,
+                "agent_config": agent_config,
+                "system_instruction": instructions if sent_only_instructions else system_instruction,
+                "base_environment": base_environment,
+                "description": description,
+                "tools": tools,
+            }
+        ),
+        drop_invalid_fields,
+        MappingProxyType(
+            {
+                "id": "name" if id is None else "id",
+                "system_instruction": "instructions" if sent_only_instructions else "system_instruction",
+            }
+        ),
+    )
+
+
 @client
 async def acreate(
-    name: str,
+    id: str | None = None,
     base_agent: str | None = None,
     instructions: str | None = None,
     base_environment: InteractionEnvironment | None = None,
@@ -104,6 +216,12 @@ async def acreate(
     extra_headers: dict[str, str] | None = None,
     extra_body: Mapping[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
+    agent_config: Mapping[str, object] | None = None,
+    system_instruction: str | None = None,
+    description: str | None = None,
+    tools: Sequence[Mapping[str, object]] | None = None,
     **kwargs,
 ) -> AgentCreateResponse:
     """Async: Create a managed agent on the provider side."""
@@ -113,7 +231,7 @@ async def acreate(
         kwargs["acreate_agent"] = True
         func: Final = partial(
             create,
-            name=name,
+            id=id,
             base_agent=base_agent,
             instructions=instructions,
             base_environment=base_environment,
@@ -121,6 +239,11 @@ async def acreate(
             extra_headers=extra_headers,
             extra_body=extra_body,
             timeout=timeout,
+            name=name,
+            agent_config=agent_config,
+            system_instruction=system_instruction,
+            description=description,
+            tools=tools,
             **kwargs,
         )
         ctx: Final = contextvars.copy_context()
@@ -130,7 +253,7 @@ async def acreate(
         return init_response
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider or "gemini",
             original_exception=e,
             completion_kwargs=local_vars,
@@ -140,7 +263,7 @@ async def acreate(
 
 @client
 def create(
-    name: str,
+    id: str | None = None,
     base_agent: str | None = None,
     instructions: str | None = None,
     base_environment: InteractionEnvironment | None = None,
@@ -148,40 +271,61 @@ def create(
     extra_headers: dict[str, str] | None = None,
     extra_body: Mapping[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
+    agent_config: Mapping[str, object] | None = None,
+    system_instruction: str | None = None,
+    description: str | None = None,
+    tools: Sequence[Mapping[str, object]] | None = None,
     **kwargs,
 ) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
     """
     Sync: Create a managed agent on the provider side.
 
     Args:
-        name: Name for the agent (required).
-        base_agent: Base agent to derive from (e.g. "waverunner").
-        instructions: System instructions for the agent.
+        id: Identifier for the agent. Exactly one of ``id`` or its alias ``name`` is required.
+        base_agent: Base agent to derive from (e.g. "antigravity-preview-09-2026").
+        instructions: Alias of ``system_instruction``.
         base_environment: Environment to fork from — an env_id string or a
             dict like ``{"type": "remote", "sources": [...]}``.
         custom_llm_provider: Provider to use, e.g. "gemini".
         extra_headers: Additional HTTP headers.
         extra_body: Additional request body fields.
         timeout: Request timeout.
+        name: Alias of ``id``.
+        agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``.
+        system_instruction: System instruction for the agent.
+        description: Description of the agent for developers.
+        tools: Tools available to the agent, e.g. ``[{"type": "code_execution"}]``.
         **kwargs: Forwarded to GenericLiteLLMParams (api_key, api_base, etc.).
     """
     local_vars: Final = locals()
-    custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
+    custom_llm_provider = custom_llm_provider or "gemini"
     try:
         _is_async: Final = kwargs.pop("acreate_agent", False) is True
-        if base_agent is not None:
-            kwargs["base_agent"] = base_agent
-        if instructions is not None:
-            kwargs["instructions"] = instructions
-        if base_environment is not None:
-            kwargs["base_environment"] = base_environment
+        create_request: Final = _create_request_or_error(
+            id=id,
+            name=name,
+            base_agent=base_agent,
+            agent_config=agent_config,
+            system_instruction=system_instruction,
+            instructions=instructions,
+            base_environment=base_environment,
+            description=description,
+            tools=tools,
+            drop_invalid_fields=drops_unsupported_params(kwargs.get("drop_params")),
+        )
+        if isinstance(create_request, _InvalidAgentRequest):
+            raise litellm.BadRequestError(
+                message=create_request.message, model=id or name or "", llm_provider=custom_llm_provider
+            )
         kwargs.setdefault("custom_llm_provider", custom_llm_provider)
         litellm_params: Final = GenericLiteLLMParams(**kwargs)
-        logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "create_agent", {})
+        logging_obj: Final = _make_logging_obj(kwargs, create_request.id, custom_llm_provider, "create_agent", {})
         config: Final = _get_agents_api_config(custom_llm_provider)
         return agents_http_handler.create_agent(
             agents_api_config=config,
-            name=name,
+            create_request=create_request,
             litellm_params=litellm_params,
             logging_obj=logging_obj,
             extra_headers=extra_headers,
@@ -191,7 +335,7 @@ def create(
         )
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider,
             original_exception=e,
             completion_kwargs=local_vars,
@@ -247,7 +391,7 @@ def list(
 ) -> AgentListResponse | Coroutine[object, object, AgentListResponse]:
     """Sync: List all agents on the provider side."""
     local_vars: Final = locals()
-    custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
+    custom_llm_provider = custom_llm_provider or "gemini"
     try:
         _is_async: Final = kwargs.pop("alist_agents", False) is True
         kwargs.setdefault("custom_llm_provider", custom_llm_provider)
@@ -279,19 +423,22 @@ def list(
 
 @client
 async def aget(
-    name: str,
+    id: str | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, str] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
     **kwargs,
 ) -> AgentCreateResponse:
-    """Async: Get a specific agent by name."""
+    """Async: Get a specific agent by id (``name`` is an alias of ``id``)."""
     local_vars: Final = locals()
     try:
         loop: Final = asyncio.get_event_loop()
         kwargs["aget_agent"] = True
         func: Final = partial(
             get,
+            id=id,
             name=name,
             custom_llm_provider=custom_llm_provider or "gemini",
             extra_headers=extra_headers,
@@ -305,7 +452,7 @@ async def aget(
         return init_response
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider or "gemini",
             original_exception=e,
             completion_kwargs=local_vars,
@@ -315,24 +462,27 @@ async def aget(
 
 @client
 def get(
-    name: str,
+    id: str | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, str] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
     **kwargs,
 ) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
-    """Sync: Get a specific agent by name."""
+    """Sync: Get a specific agent by id (``name`` is an alias of ``id``)."""
     local_vars: Final = locals()
-    custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
+    custom_llm_provider = custom_llm_provider or "gemini"
     try:
         _is_async: Final = kwargs.pop("aget_agent", False) is True
+        agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
         kwargs.setdefault("custom_llm_provider", custom_llm_provider)
         litellm_params: Final = GenericLiteLLMParams(**kwargs)
-        logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "get_agent", {"name": name})
+        logging_obj: Final = _make_logging_obj(kwargs, agent_id, custom_llm_provider, "get_agent", {"name": agent_id})
         config: Final = _get_agents_api_config(custom_llm_provider)
         return agents_http_handler.get_agent(
             agents_api_config=config,
-            name=name,
+            name=agent_id,
             litellm_params=litellm_params,
             logging_obj=logging_obj,
             extra_headers=extra_headers,
@@ -341,7 +491,7 @@ def get(
         )
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider,
             original_exception=e,
             completion_kwargs=local_vars,
@@ -356,19 +506,22 @@ def get(
 
 @client
 async def adelete(
-    name: str,
+    id: str | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, str] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
     **kwargs,
 ) -> AgentDeleteResult:
-    """Async: Delete a specific agent by name."""
+    """Async: Delete a specific agent by id (``name`` is an alias of ``id``)."""
     local_vars: Final = locals()
     try:
         loop: Final = asyncio.get_event_loop()
         kwargs["adelete_agent"] = True
         func: Final = partial(
             delete,
+            id=id,
             name=name,
             custom_llm_provider=custom_llm_provider or "gemini",
             extra_headers=extra_headers,
@@ -382,7 +535,7 @@ async def adelete(
         return init_response
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider or "gemini",
             original_exception=e,
             completion_kwargs=local_vars,
@@ -392,24 +545,29 @@ async def adelete(
 
 @client
 def delete(
-    name: str,
+    id: str | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, str] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
     **kwargs,
 ) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]:
-    """Sync: Delete a specific agent by name."""
+    """Sync: Delete a specific agent by id (``name`` is an alias of ``id``)."""
     local_vars: Final = locals()
-    custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
+    custom_llm_provider = custom_llm_provider or "gemini"
     try:
         _is_async: Final = kwargs.pop("adelete_agent", False) is True
+        agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
         kwargs.setdefault("custom_llm_provider", custom_llm_provider)
         litellm_params: Final = GenericLiteLLMParams(**kwargs)
-        logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "delete_agent", {"name": name})
+        logging_obj: Final = _make_logging_obj(
+            kwargs, agent_id, custom_llm_provider, "delete_agent", {"name": agent_id}
+        )
         config: Final = _get_agents_api_config(custom_llm_provider)
         return agents_http_handler.delete_agent(
             agents_api_config=config,
-            name=name,
+            name=agent_id,
             litellm_params=litellm_params,
             logging_obj=logging_obj,
             extra_headers=extra_headers,
@@ -418,7 +576,7 @@ def delete(
         )
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider,
             original_exception=e,
             completion_kwargs=local_vars,
@@ -433,19 +591,22 @@ def delete(
 
 @client
 async def alist_versions(
-    name: str,
+    id: str | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, str] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
     **kwargs,
 ) -> AgentVersionsResponse:
-    """Async: List versions of a specific agent."""
+    """Async: List versions of a specific agent by id (``name`` is an alias of ``id``)."""
     local_vars: Final = locals()
     try:
         loop: Final = asyncio.get_event_loop()
         kwargs["alist_agent_versions"] = True
         func: Final = partial(
             list_versions,
+            id=id,
             name=name,
             custom_llm_provider=custom_llm_provider or "gemini",
             extra_headers=extra_headers,
@@ -459,7 +620,7 @@ async def alist_versions(
         return init_response
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider or "gemini",
             original_exception=e,
             completion_kwargs=local_vars,
@@ -469,24 +630,29 @@ async def alist_versions(
 
 @client
 def list_versions(
-    name: str,
+    id: str | None = None,
     custom_llm_provider: str | None = None,
     extra_headers: dict[str, str] | None = None,
     timeout: float | httpx.Timeout | None = None,
+    *,
+    name: str | None = None,
     **kwargs,
 ) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]:
-    """Sync: List versions of a specific agent."""
+    """Sync: List versions of a specific agent by id (``name`` is an alias of ``id``)."""
     local_vars: Final = locals()
-    custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
+    custom_llm_provider = custom_llm_provider or "gemini"
     try:
         _is_async: Final = kwargs.pop("alist_agent_versions", False) is True
+        agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
         kwargs.setdefault("custom_llm_provider", custom_llm_provider)
         litellm_params: Final = GenericLiteLLMParams(**kwargs)
-        logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "list_agent_versions", {"name": name})
+        logging_obj: Final = _make_logging_obj(
+            kwargs, agent_id, custom_llm_provider, "list_agent_versions", {"name": agent_id}
+        )
         config: Final = _get_agents_api_config(custom_llm_provider)
         return agents_http_handler.list_agent_versions(
             agents_api_config=config,
-            name=name,
+            name=agent_id,
             litellm_params=litellm_params,
             logging_obj=logging_obj,
             extra_headers=extra_headers,
@@ -495,7 +661,7 @@ def list_versions(
         )
     except Exception as e:
         raise litellm.exception_type(
-            model=name,
+            model=id or name or "",
             custom_llm_provider=custom_llm_provider,
             original_exception=e,
             completion_kwargs=local_vars,

litellm/interactions/main.py

@@ -49,6 +49,7 @@ from litellm.interactions.utils import (
     InteractionsAPIRequestUtils,
     get_provider_interactions_api_config,
 )
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.types.interactions import (
     CancelInteractionResult,
@@ -100,6 +101,8 @@ async def acreate(
     timeout: float | httpx.Timeout | None = None,
     # LiteLLM params
     custom_llm_provider: str | None = None,
+    *,
+    agent_config: Mapping[str, object] | None = None,
     **kwargs,
 ) -> InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]:
     """
@@ -121,6 +124,7 @@ async def acreate(
             string, or a config object such as
             ``{"type": "remote", "sources": [...]}`` /
             ``{"type": "remote", "network": {...}}``
+        agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``
         response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
         response_format: JSON schema for response format
         response_mime_type: MIME type of the response
@@ -155,6 +159,7 @@ async def acreate(
             store=store,
             background=background,
             environment=environment,
+            agent_config=agent_config,
             response_modalities=response_modalities,
             response_format=response_format,
             response_mime_type=response_mime_type,
@@ -225,6 +230,8 @@ def create(
     timeout: float | httpx.Timeout | None = None,
     # LiteLLM params
     custom_llm_provider: str | None = None,
+    *,
+    agent_config: Mapping[str, object] | None = None,
     **kwargs,
 ) -> (
     InteractionsAPIResponse
@@ -250,6 +257,7 @@ def create(
             string, or a config object such as
             ``{"type": "remote", "sources": [...]}`` /
             ``{"type": "remote", "network": {...}}``
+        agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``
         response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
         response_format: JSON schema for response format
         response_mime_type: MIME type of the response
@@ -300,6 +308,15 @@ def create(
         # Check if this is a bridge provider (litellm_responses) - similar to responses API
         # Either provider is explicitly "litellm_responses" or no config found (bridge to responses)
         if custom_llm_provider == "litellm_responses" or interactions_api_config is None:
+            if optional_params.get("agent_config") is not None and not drops_unsupported_params(
+                kwargs.get("drop_params")
+            ):
+                raise litellm.UnsupportedParamsError(
+                    message=f"'agent_config' is not supported for {custom_llm_provider} interactions. "
+                    "Set drop_params to drop it.",
+                    llm_provider=custom_llm_provider,
+                    model=model,
+                )
             # Bridge to litellm.responses() for non-native providers
             from litellm.interactions.litellm_responses_transformation.handler import (
                 LiteLLMResponsesInteractionsHandler,

litellm/interactions/streaming_iterator.py

@@ -21,6 +21,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
 from litellm.litellm_core_utils.thread_pool_executor import executor
 from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
 from litellm.types.interactions import (
     InteractionsAPIStreamingResponse,
 )
@@ -145,7 +146,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
             litellm_metadata=litellm_metadata,
             custom_llm_provider=custom_llm_provider,
         )
-        self.stream_iterator = response.aiter_lines()
+        self.stream_iterator = aiter_sse_lines(response.aiter_text())
 
     def __aiter__(self):
         return self
@@ -212,7 +213,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator)
             litellm_metadata=litellm_metadata,
             custom_llm_provider=custom_llm_provider,
         )
-        self.stream_iterator = response.iter_lines()
+        self.stream_iterator = iter_sse_lines(response.iter_text())
 
     def __iter__(self):
         return self

litellm/litellm_core_utils/README.md

@@ -11,4 +11,4 @@ Core files:
 - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
 - `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
 
-Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
+Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.

litellm/litellm_core_utils/audio_utils/utils.py

@@ -2,8 +2,12 @@
 Utils used for litellm.transcription() and litellm.atranscription()
 """
 
+import base64
+import binascii
 import hashlib
+import math
 import os
+from collections.abc import Sequence
 from dataclasses import dataclass
 from typing import Final
 
@@ -331,6 +335,45 @@ def calculate_request_duration(file: FileTypes) -> float | None:
         return None
 
 
+DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
+AUDIO_TOKENS_PER_SECOND: Final = 10
+AUDIO_BYTES_PER_TOKEN: Final = 1600
+_MAX_AUDIO_ESTIMATE_BYTES: Final = 16 * 1024 * 1024
+
+
+def _audio_token_rate(model: str) -> int | None:
+    if model.startswith(
+        ("gemini-", "gemini/gemini-", "vertex_ai/gemini-", "vertex_ai_beta/gemini-", "openrouter/google/gemini-")
+    ):
+        family: Final = model.rsplit("/", 1)[-1]
+        return 25 if family.startswith(("gemini-3-", "gemini-3.")) else 32
+    if model.startswith(("gpt-", "openai/gpt-", "azure/gpt-", "openrouter/openai/gpt-")):
+        return AUDIO_TOKENS_PER_SECOND
+    return None
+
+
+def estimate_audio_tokens(data: str | None, models: Sequence[str] = (), fallback_model: str | None = None) -> int:
+    """Estimate audio tokens from container duration within a bounded decoding budget."""
+    if not data:
+        return DEFAULT_AUDIO_TOKEN_ESTIMATE
+    fallback_estimate: Final = max(len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
+    padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
+    if len(data) * 3 // 4 - padding > _MAX_AUDIO_ESTIMATE_BYTES:
+        return fallback_estimate
+    try:
+        duration: Final = calculate_request_duration(base64.b64decode(data, validate=True))
+    except (ValueError, binascii.Error):
+        return fallback_estimate
+    if duration is None or not math.isfinite(duration) or duration <= 0:
+        return fallback_estimate
+    fallback_rate: Final = _audio_token_rate(fallback_model) if fallback_model is not None else None
+    rate: Final = max(
+        (_audio_token_rate(model) or fallback_rate or AUDIO_TOKENS_PER_SECOND for model in models),
+        default=fallback_rate or AUDIO_TOKENS_PER_SECOND,
+    )
+    return max(1, math.ceil(duration * rate))
+
+
 DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg"
 
 

litellm/litellm_core_utils/cli_token_utils.py

@@ -353,9 +353,9 @@ def is_cli_token_fresh(
     buffer_hours: float = CLI_TOKEN_FRESHNESS_BUFFER_SECONDS / 3600,
 ) -> bool:
     """Check whether a cached CLI token is still within its expiration window.
-    Used by `lite auth print-token` to fail fast, without a network round trip,
-    once the cached token is past `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce`
-    credential carries its own `expires_at`, which is authoritative when present."""
+    Used by `lite auth print-token` to skip asking the proxy while the cached token
+    is younger than `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce` credential
+    carries its own `expires_at`, which is authoritative when present."""
     from litellm.constants import CLI_JWT_EXPIRATION_HOURS
 
     expires_at: Final = (

litellm/litellm_core_utils/core_helpers.py

@@ -266,6 +266,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "tool_use": "tool_calls",
     "refusal": "content_filter",
     "compaction": "length",
+    "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
     "ERROR_TOXIC": "content_filter",

litellm/litellm_core_utils/get_litellm_params.py

@@ -75,6 +75,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "rpm",
             "itpm",
             "otpm",
+            "tpd",
+            "rpd",
             "use_xai_oauth",
             "chatgpt_auth_file",
             "github_copilot_auth_type",

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -437,7 +437,6 @@ def get_llm_provider(
             model in litellm.open_ai_chat_completion_models
             or "ft:gpt-3.5-turbo" in model
             or "ft:gpt-4" in model  # catches ft:gpt-4-0613, ft:gpt-4o
-            or model in litellm.openai_image_generation_models
             or model.startswith("gpt-image")
             or model in litellm.openai_video_generation_models
         ):

litellm/litellm_core_utils/litellm_logging.py

@@ -842,6 +842,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self._async_success_enqueued: bool = False
         self._sync_success_scheduled: bool = False
         self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
+        self._stream_failure_claimed: bool = False
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
     def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
@@ -2561,6 +2562,12 @@ class Logging(LiteLLMLoggingBaseClass):
                 },
             )
 
+    def claim_stream_failure_report(self) -> bool:
+        if self._stream_failure_claimed:
+            return False
+        self._stream_failure_claimed = True
+        return True
+
     def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
         """Bill a fully streamed response on the failure log when a post-call hook rejects it."""
         usage: Final = getattr(assembled, "usage", None)
@@ -2751,7 +2758,7 @@ class Logging(LiteLLMLoggingBaseClass):
         if entry is None or not is_str_keyed_dict(litellm_params):
             return
         metadata: Final[object] = litellm_params.get("metadata")
-        bucket: Final = metadata if is_str_keyed_dict(metadata) else {}
+        bucket: Final = {**metadata} if is_str_keyed_dict(metadata) else {}
         litellm_params["metadata"] = bucket
         with_guardrail_entry(bucket, entry)
 
@@ -3474,7 +3481,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             start_time=start_time,
                             end_time=end_time,
                             print_verbose=print_verbose,
-                            level=LogfireLevel.INFO.value,
+                            level=LogfireLevel.INFO,
                         )
 
                     if callback == "lunary" and lunaryLogger is not None:
@@ -4435,7 +4442,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             response_obj=result,
                             start_time=start_time,
                             end_time=end_time,
-                            level=LogfireLevel.ERROR.value,
+                            level=LogfireLevel.ERROR,
                             print_verbose=print_verbose,
                         )
 

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from datetime import datetime, timezone, tzinfo
 from types import MappingProxyType
-from typing import Final, Literal, TypedDict, cast
+from typing import Final, Literal, Protocol, TypedDict, cast
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 from typing_extensions import ReadOnly
@@ -2085,6 +2085,41 @@ def calculate_image_response_web_search_cost(
     )
 
 
+class _ModelPricedImageCostCalculator(Protocol):
+    def __call__(self, model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: ...
+
+
+def _model_priced_image_cost_calculator(
+    custom_llm_provider: str | None, model: str
+) -> _ModelPricedImageCostCalculator | None:
+    match custom_llm_provider:
+        case litellm.LlmProviders.RECRAFT.value:
+            from litellm.llms.recraft.cost_calculator import cost_calculator as recraft_image_cost_calculator
+
+            return recraft_image_cost_calculator
+        case litellm.LlmProviders.AIML.value:
+            from litellm.llms.aiml.image_generation.cost_calculator import cost_calculator as aiml_image_cost_calculator
+
+            return aiml_image_cost_calculator
+        case litellm.LlmProviders.COMETAPI.value:
+            from litellm.llms.cometapi.image_generation.cost_calculator import (
+                cost_calculator as cometapi_image_cost_calculator,
+            )
+
+            return cometapi_image_cost_calculator
+        case litellm.LlmProviders.RUNWAYML.value:
+            from litellm.llms.runwayml.cost_calculator import cost_calculator as runwayml_image_cost_calculator
+
+            return runwayml_image_cost_calculator
+        case litellm.LlmProviders.TENCENT.value:
+            from litellm.llms.tencent.cost_calculator import image_cost_calculator as tencent_image_cost_calculator
+            from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
+
+            return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
+        case _:
+            return None
+
+
 class CostCalculatorUtils:
     @staticmethod
     def call_type_has_image_response(call_type: str) -> bool:
@@ -2131,9 +2166,6 @@ class CostCalculatorUtils:
         from litellm.llms.gemini.image_generation.cost_calculator import (
             cost_calculator as gemini_image_cost_calculator,
         )
-        from litellm.llms.recraft.cost_calculator import (
-            cost_calculator as recraft_image_cost_calculator,
-        )
         from litellm.llms.vertex_ai.image_generation.cost_calculator import (
             cost_calculator as vertex_ai_image_cost_calculator,
         )
@@ -2146,7 +2178,10 @@ class CostCalculatorUtils:
         )
         resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
         pricing: Final = deployment_pricing(model_info)
+        model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
 
+        if model_priced_calculator is not None:
+            return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)
         if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
             if isinstance(completion_response, ImageResponse):
                 return vertex_ai_image_cost_calculator(
@@ -2164,36 +2199,6 @@ class CostCalculatorUtils:
                     optional_params=optional_params,
                 )
             raise TypeError("completion_response must be of type ImageResponse for bedrock image cost calculation")
-        elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
-            from litellm.llms.recraft.cost_calculator import (
-                cost_calculator as recraft_image_cost_calculator,
-            )
-
-            return recraft_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
-        elif custom_llm_provider == litellm.LlmProviders.AIML.value:
-            from litellm.llms.aiml.image_generation.cost_calculator import (
-                cost_calculator as aiml_image_cost_calculator,
-            )
-
-            return aiml_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
-        elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
-            from litellm.llms.cometapi.image_generation.cost_calculator import (
-                cost_calculator as cometapi_image_cost_calculator,
-            )
-
-            return cometapi_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
         elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
             if call_type in (
                 CallTypes.image_edit.value,
@@ -2237,16 +2242,6 @@ class CostCalculatorUtils:
                 optional_params=optional_params,
                 model_info=pricing,
             )
-        elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
-            from litellm.llms.runwayml.cost_calculator import (
-                cost_calculator as runwayml_image_cost_calculator,
-            )
-
-            return runwayml_image_cost_calculator(
-                model=model,
-                image_response=completion_response,
-                model_info=pricing,
-            )
         elif (
             custom_llm_provider == litellm.LlmProviders.OPENAI.value
             or custom_llm_provider == litellm.LlmProviders.AZURE.value

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -9,6 +9,7 @@ import json
 import mimetypes
 import re
 from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from graphlib import CycleError, TopologicalSorter
 from itertools import chain, groupby, islice
 from os import PathLike
@@ -20,8 +21,8 @@ from pydantic import TypeAdapter
 
 import litellm
 from litellm import verbose_logger
+from litellm.litellm_core_utils.batch_utils import InMemoryFile
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
-from litellm.router_utils.batch_utils import InMemoryFile
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionAssistantMessage,
@@ -236,6 +237,160 @@ _IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image
 _IMAGE_SCAN_MAX_DEPTH: Final = 4
 
 
+@dataclass(frozen=True, slots=True)
+class _TPMMediaProjection:
+    content: object
+    media: tuple[Mapping[str, object], ...] = ()
+
+
+def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]:
+    return {"type": "image_url", "image_url": {"url": url, "detail": detail}}
+
+
+def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]:
+    return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()}
+
+
+def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection:
+    from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
+
+    nested: Final = part.get("type") == "file"
+    fields: Final = as_str_mapping(part.get("file")) if nested else part
+    if fields is None:
+        return _TPMMediaProjection(part)
+    raw_data: Final = fields.get("file_data")
+    data: Final = raw_data if isinstance(raw_data, str) else ""
+    declared: Final = next(
+        (
+            value
+            for key in ("format", "mime_type", "content_type")
+            if isinstance(value := fields.get(key), str) and value
+        ),
+        "",
+    )
+    mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower()
+    if not mime.startswith(("image/", "audio/")):
+        return _TPMMediaProjection(part)
+    raw_url: Final = fields.get("file_url") or fields.get("file_id")
+    url: Final = data or (raw_url if isinstance(raw_url, str) else "")
+    header_end: Final = data.find(",", 0, 256)
+    encoded: Final = (
+        data[header_end + 1 :]
+        if header_end >= 0 and data[:header_end].endswith(";base64")
+        else ""
+        if data.startswith(("data:", "https://", "http://", "gs://"))
+        else data
+    )
+    media: Final = (
+        _tpm_image_part(
+            url
+            if not data or url.startswith(("data:", "https://", "http://", "gs://"))
+            else f"data:{mime};base64,{url}",
+            fields.get("detail", "auto"),
+        )
+        if mime.startswith("image/")
+        else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}}
+    )
+    projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url"))
+    return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,))
+
+
+def _tpm_media_leaf(value: object) -> _TPMMediaProjection:
+    part: Final = as_str_mapping(value)
+    if part is None:
+        return _TPMMediaProjection(value)
+    kind: Final = part.get("type")
+    if kind in ("file", "input_file"):
+        return _tpm_file_media(part)
+    if kind == "input_audio":
+        audio: Final = as_str_mapping(part.get("input_audio"))
+        return _TPMMediaProjection(
+            {**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value,
+            ({"type": "input_audio", "input_audio": part.get("input_audio")},),
+        )
+    if kind in ("image_url", "input_image", "image", "computer_screenshot"):
+        image: Final = as_str_mapping(part.get("image_url"))
+        source: Final = as_str_mapping(part.get("source"))
+        canonical: Final = _tpm_image_part(
+            _image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "",
+            (image or part).get("detail", part.get("detail", "auto")),
+        )
+        projected: Final = (
+            {**part, "image_url": _tpm_blank_fields(image, ("url",))}
+            if image is not None and kind in ("image_url", "input_image", "computer_screenshot")
+            else {**part, "source": _tpm_blank_fields(source, ("data", "url"))}
+            if source is not None and kind == "image" and source.get("type") in ("base64", "url")
+            else _tpm_blank_fields(part, ("image_url", "url"))
+        )
+        return _TPMMediaProjection(projected, (canonical,))
+    return _TPMMediaProjection(value)
+
+
+def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection:
+    if not isinstance(value, (list, tuple)):
+        return _TPMMediaProjection(value)
+    parts: Final = cast(Sequence[object], value)  # cast-ok: finite JSON content arrays
+    projected: Final = tuple(map(split, parts))
+    media: Final = tuple(chain.from_iterable(part.media for part in projected))
+    content: Final = tuple(part.content for part in projected)
+    return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media)
+
+
+def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection:
+    return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media)
+
+
+def _tpm_document_media(value: object) -> _TPMMediaProjection:
+    part: Final = as_str_mapping(value)
+    source: Final = as_str_mapping(part.get("source")) if part is not None else None
+    if part is None or part.get("type") != "document" or source is None or source.get("type") != "content":
+        return _tpm_media_leaf(value)
+    content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf)
+    return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content))
+
+
+def _tpm_tool_result_media(value: object) -> _TPMMediaProjection:
+    part: Final = as_str_mapping(value)
+    if part is None or part.get("type") != "tool_result":
+        return _tpm_document_media(value)
+    return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media))
+
+
+def _tpm_code_output_media(value: object) -> _TPMMediaProjection:
+    part: Final = as_str_mapping(value)
+    url: Final = part.get("url") if part is not None else None
+    if part is None or part.get("type") != "image" or not isinstance(url, str) or not url:
+        return _TPMMediaProjection(value)
+    return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),))
+
+
+def _tpm_input_media(value: object) -> _TPMMediaProjection:
+    part: Final = as_str_mapping(value)
+    if part is None:
+        return _TPMMediaProjection(value)
+    kind: Final = part.get("type")
+    if kind in ("function_call_output", "custom_tool_call_output"):
+        return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media))
+    if kind == "computer_call_output":
+        return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output")))
+    if kind == "code_interpreter_call":
+        return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media))
+    result: Final = part.get("result")
+    if kind == "image_generation_call" and isinstance(result, str) and result:
+        return _TPMMediaProjection(
+            _tpm_blank_fields(part, ("result",)),
+            (_tpm_image_part("data:application/octet-stream;base64," + result),),
+        )
+    if kind == "message" or (kind is None and isinstance(part.get("role"), str)):
+        return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media))
+    return _tpm_tool_result_media(value)
+
+
+def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
+    projection: Final = _tpm_media_list(value, _tpm_input_media)
+    return projection.content, projection.media
+
+
 def _content_parts_contain_image(parts: Sequence[object]) -> bool:
     """Depth-bounded frontier walk over nested content lists, iterative because the repo bans
     recursion; an Anthropic tool_result nests its image parts exactly one level down."""

litellm/litellm_core_utils/safe_json_dumps.py

@@ -34,7 +34,7 @@ def safe_json_structure(
     def _transform(key: str | None, value: str) -> str:
         return value if value_transform is None else value_transform(key, value)
 
-    def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> Any:
+    def _serialize(obj: object, seen: set[int], depth: int, key: str | None = None) -> object:
         # Check for maximum depth.
         if depth > max_depth:
             return "MaxDepthExceeded"
@@ -48,32 +48,32 @@ def safe_json_structure(
         if id(obj) in seen:
             return "CircularReference Detected"
         seen.add(id(obj))
-        result: dict | list | tuple | set | str
         if isinstance(obj, dict):
-            result = {}
-            for k, v in obj.items():
-                if isinstance(k, (str)):
-                    clean_k = k.replace("\x00", "") if "\x00" in k else k
-                    result[clean_k] = _serialize(v, seen, depth + 1, clean_k)
+            mapping_result: Final = {
+                (clean_k := k.replace("\x00", "") if "\x00" in k else k): _serialize(v, seen, depth + 1, clean_k)
+                for k, v in obj.items()
+                if isinstance(k, str)
+            }
             seen.remove(id(obj))
-            return result
+            return mapping_result
         elif isinstance(obj, list):
-            result = [_serialize(item, seen, depth + 1, key) for item in obj]
+            list_result: Final = [_serialize(item, seen, depth + 1, key) for item in obj]
             seen.remove(id(obj))
-            return result
+            return list_result
         elif isinstance(obj, tuple):
-            result = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
+            tuple_result: Final = tuple(_serialize(item, seen, depth + 1, key) for item in obj)
             seen.remove(id(obj))
-            return result
+            return tuple_result
         elif isinstance(obj, set):
-            result = sorted([_serialize(item, seen, depth + 1, key) for item in obj])
+            sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj]
+            set_result: Final = sorted(sortable_values)
             seen.remove(id(obj))
-            return result
+            return set_result
         elif isinstance(obj, BaseModel):
             dumped: Final = obj.model_dump()
-            result = _serialize(dumped, seen, depth + 1, key)
+            model_result: Final = _serialize(dumped, seen, depth + 1, key)
             seen.remove(id(obj))
-            return result
+            return model_result
         else:
             # Fall back to string conversion for non-serializable objects.
             try:

litellm/litellm_core_utils/streaming_handler.py

@@ -1263,7 +1263,7 @@ class CustomStreamWrapper:
                 )
 
             if _is_delta_empty:
-                model_response.choices[0].delta = Delta(content=None)  # ensure empty delta chunk returned
+                model_response.choices[0].delta = Delta(content=self._close_open_think_block())
                 # get any function call arguments
                 model_response.choices[0].finish_reason = map_finish_reason(
                     finish_reason=self.received_finish_reason
@@ -1292,21 +1292,26 @@ class CustomStreamWrapper:
         if self.merge_reasoning_content_in_choices is not True:
             return
         delta: Final = model_response.choices[0].delta
-        reasoning_content: Final = getattr(delta, "reasoning_content", None)
+        reasoning_content: Final = getattr(delta, "reasoning_content", None) or ""
         content: Final = delta.content or ""
-        if reasoning_content:
-            self._emitted_disqualifying_content = True
-            closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
-            opening: Final = "" if self.sent_first_thinking_block else "<think>"
-            delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
-            self.sent_first_thinking_block = True
-            self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
-        elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
-            delta.content = "</think>" + content
-            self.sent_last_thinking_block = True
-
+        ends_thinking: Final = bool(content or delta.tool_calls or delta.function_call)
+        inside_think_block: Final = self.sent_first_thinking_block and not self.sent_last_thinking_block
+        opening: Final = "<think>" if reasoning_content and not inside_think_block else ""
+        closing: Final = "</think>" if ends_thinking and (reasoning_content or inside_think_block) else ""
         if hasattr(delta, "reasoning_content"):
             del delta.reasoning_content
+        if not reasoning_content and not closing:
+            return
+        delta.content = opening + reasoning_content + closing + content
+        self.sent_first_thinking_block = True
+        self.sent_last_thinking_block = bool(closing)
+        self._emitted_disqualifying_content = self._emitted_disqualifying_content or bool(reasoning_content)
+
+    def _close_open_think_block(self) -> str | None:
+        if not self.sent_first_thinking_block or self.sent_last_thinking_block:
+            return None
+        self.sent_last_thinking_block = True
+        return "</think>"
 
     def _dispatch_provider_chunk(
         self,
@@ -1557,6 +1562,12 @@ class CustomStreamWrapper:
         else:  # openai / azure chat model
             if isinstance(chunk, ChatCompletionChunk):
                 self._openai_handler_receives_chat_chunks = True
+                if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
+                    raise litellm.InternalServerError(
+                        message="Upstream generation aborted (finish_reason=abort)",
+                        model=chunk.model,
+                        llm_provider=self.custom_llm_provider,
+                    )
             if self.custom_llm_provider in [
                 LlmProviders.AZURE.value,
                 LlmProviders.AZURE_AI.value,
@@ -1926,8 +1937,20 @@ class CustomStreamWrapper:
             model_response.choices[0].finish_reason == "stop" and self.tool_call
         ):  # don't overwrite for other - potential error finish reasons
             model_response.choices[0].finish_reason = "tool_calls"
+        close_tag: Final = self._close_open_think_block()
+        if close_tag is not None:
+            model_response.choices[0].delta.content = close_tag
         return model_response
 
+    def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
+        if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
+            return processed_chunk
+        if processed_chunk.choices[0].delta.content is None:
+            return None
+        return processed_chunk.model_copy(
+            update={"choices": [processed_chunk.choices[0].model_copy(update={"finish_reason": None})]}
+        )
+
     def _record_usage_only_chunk(self, model_response: "ModelResponseStream") -> None:
         """
         Keep provider usage-only chunks (e.g. OpenRouter's post-finish chunk, which carries a
@@ -2167,10 +2190,9 @@ class CustomStreamWrapper:
             else:
                 self.sent_last_chunk = True
                 processed_chunk: Final = self.finish_reason_handler()
-                # The logged response is built from self.chunks; keep a finish_reason the provider sent on its
-                # last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
-                if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
-                    self.chunks.append(processed_chunk)
+                final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
+                if final_chunk_for_logging is not None:
+                    self.chunks.append(final_chunk_for_logging)
                 if self.stream_options is None:
                     self._set_final_chunk_usage(processed_chunk)
                 ## LOGGING
@@ -2400,10 +2422,9 @@ class CustomStreamWrapper:
         else:
             self.sent_last_chunk = True
             processed_chunk: Final = self.finish_reason_handler()
-            # The logged response is built from self.chunks; keep a finish_reason the provider sent on its
-            # last content chunk (stripped there), but never add the synthetic "stop" used when it sent none.
-            if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
-                self.chunks.append(processed_chunk)
+            final_chunk_for_logging: Final = self._final_chunk_for_logging(processed_chunk)
+            if final_chunk_for_logging is not None:
+                self.chunks.append(final_chunk_for_logging)
             if self.stream_options is None:
                 self._set_final_chunk_usage(processed_chunk)
             # see sync __next__'s sibling branch: deliberately do NOT restore

litellm/litellm_core_utils/token_counter.py

@@ -3,9 +3,12 @@
 import base64
 import io
 import math
+import re
 import struct
+from bisect import bisect_right
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from itertools import accumulate
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal, cast
 
 import anyio
@@ -35,6 +38,7 @@ from litellm.constants import (
     TOKEN_COUNTER_MAX_EXACT_CHARS,
 )
 from litellm.litellm_core_utils.asyncify import asyncify
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
     ChatTemplateToolSection,
     chat_template_tool_section,
@@ -188,22 +192,23 @@ def resize_image_high_res(
 
 # Test the function with the given example
 def calculate_tiles_needed(
-    resized_width,
-    resized_height,
-    tile_width=MAX_TILE_WIDTH,
-    tile_height=MAX_TILE_HEIGHT,
-):
+    resized_width: int,
+    resized_height: int,
+    tile_width: int = MAX_TILE_WIDTH,
+    tile_height: int = MAX_TILE_HEIGHT,
+) -> int:
     tiles_across: Final = (resized_width + tile_width - 1) // tile_width
     tiles_down: Final = (resized_height + tile_height - 1) // tile_height
     total_tiles: Final = tiles_across * tiles_down
     return total_tiles
 
 
-def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int:
+def high_detail_image_token_upper_bound(base_tokens: int = 85, tile_tokens: int | None = None) -> int:
     largest_tile_count: Final = calculate_tiles_needed(
         MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES
     )
-    return base_tokens + (base_tokens * 2) * largest_tile_count
+    per_tile: Final = base_tokens * 2 if tile_tokens is None else tile_tokens
+    return base_tokens + per_tile * largest_tile_count
 
 
 def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]:
@@ -381,6 +386,219 @@ def calculate_img_tokens(
     return total_tokens
 
 
+_IMAGE_HEADER_BASE64_CHARS: Final = 65_536
+_MAX_IMAGE_PATCHES: Final = 30_000
+_ANTHROPIC_IMAGE_LIMITS: Final = MappingProxyType(
+    {"claude-standard": (ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX, 1568), "claude-high": (2576, 4784)}
+)
+_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType(
+    {
+        "gpt-4o": (85, 170),
+        "gpt-4.1": (85, 170),
+        "gpt-4o-mini": (2833, 5667),
+        "gpt-5": (70, 140),
+        "gpt-5.1": (70, 140),
+        "o1": (75, 150),
+        "o1-pro": (75, 150),
+        "o3": (75, 150),
+    }
+)
+_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType(
+    {
+        "gpt-6-astra": 1.2,
+        "gpt-5.6-sol": 1.2,
+        "gpt-5.6-terra": 1.2,
+        "gpt-5.6-luna": 1.2,
+        "gpt-5.5": 1.2,
+        "gpt-5.4": 1.2,
+        "gpt-5.4-mini": 1.2,
+        "gpt-5.4-nano": 1.2,
+        "gpt-5.2": 1.2,
+        "gpt-4.1-mini": 1.62,
+    }
+)
+
+
+def _inline_image_dimensions(data: str) -> tuple[int, int] | None:
+    separator: Final = data.find(",", 0, 256)
+    if not data.startswith("data:") or separator < 0 or not data[:separator].endswith(";base64"):
+        return None
+    encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS]
+    try:
+        header: Final = base64.b64decode(encoded_header, validate=True)
+        image_type: Final = get_image_type(header)
+        if image_type not in ("png", "gif", "jpeg", "webp"):
+            return None
+        if image_type == "webp" and header[12:16] not in (b"VP8X", b"VP8 ", b"VP8L"):
+            return None
+        dimensions: Final = _image_dimensions_from_bytes(header)
+    except (ValueError, TypeError, struct.error):
+        return None
+    width, height = dimensions
+    return (width, height) if width > 0 and height > 0 else None
+
+
+def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None:
+    if model in ("gpt-5.2", "gpt-4.1-mini"):
+        return None if detail == "original" else (2048, 6144)
+    original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5")
+    effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail
+    if effective_detail == "low":
+        return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None)
+    if effective_detail == "high":
+        return (65_535 if model == "gpt-6-astra" else 2048, 2500)
+    return (
+        (65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000)
+    )
+
+
+def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int:
+    dimension_scale: Final = min(1.0, max_dimension / max(width, height))
+    fitted_width: Final = max(1, round(width * dimension_scale))
+    fitted_height: Final = max(1, round(height * dimension_scale))
+    patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32)
+    if budget is None or patches <= budget:
+        return patches
+    shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height))
+    width_patches: Final = fitted_width * shrink / 32
+    height_patches: Final = fitted_height * shrink / 32
+    aligned_shrink: Final = shrink * min(
+        math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches
+    )
+    resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink))
+    resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink))
+    return math.ceil(resized_width / 32) * math.ceil(resized_height / 32)
+
+
+def _image_policy_model_name(model: str) -> str | None:
+    provider, separator, provider_model = model.partition("/")
+    if not separator:
+        return model
+    if provider == "openai":
+        return provider_model
+    if provider == "openrouter":
+        return provider_model.removeprefix("openai/") if provider_model.startswith("openai/") else None
+    if provider not in ("azure", "azure_ai"):
+        return None
+    catalog_entry: Final = as_str_mapping(
+        cast(  # cast-ok: read-only view of the legacy string-keyed registry; as_str_mapping checks the entry
+            Mapping[str, object], litellm.model_cost
+        ).get(model)
+    )
+    if catalog_entry is not None and catalog_entry.get("litellm_provider") in ("azure", "azure_ai"):
+        return provider_model.rsplit("/", 1)[-1]
+    return provider_model.removeprefix("responses/").removeprefix("o_series/").removeprefix("openai/")
+
+
+def _anthropic_image_family(model: str) -> str | None:
+    from litellm.utils import get_potential_model_names
+
+    provider, separator, _ = model.partition("/")
+    if separator and provider not in (
+        "anthropic",
+        "bedrock",
+        "bedrock_converse",
+        "bedrock_mantle",
+        "vertex_ai",
+        "vertex_ai_beta",
+        "azure_ai",
+        "openrouter",
+    ):
+        return None
+    candidates: Final = get_potential_model_names(model, provider if separator else "anthropic")
+    catalog: Final = cast(Mapping[str, object], litellm.model_cost)  # cast-ok: read-only view of the legacy registry
+    for candidate in dict.fromkeys(candidates.values()):
+        if not isinstance(candidate, str):
+            continue
+        if (entry := as_str_mapping(catalog.get(candidate))) is None or entry.get("supports_vision") is not True:
+            continue
+        if (
+            version := re.search(
+                r"(?:^|[./])claude-(?:(?:haiku|sonnet|opus)-)?(\d+)(?:[-.](\d{1,2})(?!\d))?", candidate
+            )
+        ) is None:
+            continue
+        major, minor = int(version[1]), int(version[2] or 0)
+        return "claude-high" if (major, minor) >= (4, 7) else "claude-standard"
+    return None
+
+
+def _anthropic_image_tokens(dimensions: tuple[int, int] | None, limits: tuple[int, int]) -> int:
+    max_edge, token_budget = limits
+    if dimensions is None:
+        return token_budget
+    long_edge, short_edge = max(dimensions), min(dimensions)
+    aspect_ratio: Final = long_edge / short_edge
+
+    def tokens_at_edge(edge: int) -> int:
+        return math.ceil(edge / 28) * math.ceil(max(round(edge / aspect_ratio), 1) / 28)
+
+    fitted_edge: Final = bisect_right(range(1, min(long_edge, max_edge) + 1), token_budget, key=tokens_at_edge)
+    return tokens_at_edge(max(fitted_edge, 1))
+
+
+def _image_model_family(model: str) -> str | None:
+    provider_model: Final = _image_policy_model_name(model)
+    if provider_model is None:
+        return _anthropic_image_family(model)
+    family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model)
+    canonical: Final = litellm.azure_llms.get(f"azure/{family}", family) if model.startswith("azure/") else family
+    return (
+        canonical
+        if canonical in _TILE_IMAGE_TOKEN_COSTS or canonical in _PATCH_IMAGE_MULTIPLIERS
+        else _anthropic_image_family(model)
+    )
+
+
+def _image_policy_tokens(family: str | None, dimensions: tuple[int, int] | None, detail: str) -> int:
+    if family is None:
+        return DEFAULT_IMAGE_TOKEN_COUNT
+    anthropic_limits: Final = _ANTHROPIC_IMAGE_LIMITS.get(family)
+    if anthropic_limits is not None:
+        return _anthropic_image_tokens(dimensions, anthropic_limits)
+    tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family)
+    multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family)
+    if tile_costs is not None:
+        base_tokens, tile_tokens = tile_costs
+        if detail == "original":
+            return DEFAULT_IMAGE_TOKEN_COUNT
+        if detail == "low":
+            return base_tokens
+        if dimensions is None:
+            return high_detail_image_token_upper_bound(base_tokens, tile_tokens)
+        width, height = dimensions
+        scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height))
+        tiles: Final = calculate_tiles_needed(max(1, math.floor(width * scale)), max(1, math.floor(height * scale)))
+        return base_tokens + tiles * tile_tokens
+    limits: Final = _patch_image_limits(family, detail)
+    if limits is None or multiplier is None:
+        return DEFAULT_IMAGE_TOKEN_COUNT
+    max_dimension, patch_budget = limits
+    patches: Final = (
+        min(math.ceil(max_dimension / 32) ** 2, patch_budget if patch_budget is not None else _MAX_IMAGE_PATCHES)
+        if dimensions is None
+        else _image_patch_count(*dimensions, max_dimension, patch_budget)
+    )
+    return math.ceil(patches * multiplier)
+
+
+def estimate_image_tokens(
+    model: str | Sequence[str], data: str, detail: str = "auto", *, fallback_model: str | None = None
+) -> int:
+    """Estimate image policies from at most 48 KiB of inline headers, without fetching media."""
+    if detail not in ("low", "high", "auto", "original"):
+        return DEFAULT_IMAGE_TOKEN_COUNT
+    models: Final = (model,) if isinstance(model, str) else model
+    fallback_family: Final = _image_model_family(fallback_model) if fallback_model is not None else None
+    families: Final = tuple(
+        dict.fromkeys(_image_model_family(candidate) or fallback_family for candidate in models or ("",))
+    )
+    if all(family is None for family in families):
+        return DEFAULT_IMAGE_TOKEN_COUNT
+    dimensions: Final = _inline_image_dimensions(data)
+    return max(_image_policy_tokens(family, dimensions, detail) for family in families)
+
+
 TokenCounterFunction = Callable[[str], int]
 """
 Type for a function that counts tokens in a string.

litellm/llms/aiohttp_openai/chat/transformation.py

@@ -82,4 +82,6 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
         model_response.model = _json_response.get("model")
         model_response.object = _json_response.get("object")
         model_response.system_fingerprint = _json_response.get("system_fingerprint")
+        if "service_tier" in _json_response:
+            return model_response.model_copy(update={"service_tier": _json_response.get("service_tier")})
         return model_response

litellm/llms/anthropic/chat/handler.py

@@ -38,6 +38,7 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     get_httpx_client,
 )
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
 from litellm.types.llms.anthropic import (
     ContentBlockDelta,
     ContentBlockStart,
@@ -233,7 +234,7 @@ async def make_call(
         raise AnthropicError(status_code=500, message=str(e))
 
     completion_stream: Final = ModelResponseIterator(
-        streaming_response=response.aiter_lines(),
+        streaming_response=aiter_sse_lines(response.aiter_text()),
         sync_stream=False,
         json_mode=json_mode,
         speed=speed,
@@ -304,7 +305,7 @@ def make_sync_call(
         )
 
     completion_stream: Final = ModelResponseIterator(
-        streaming_response=response.iter_lines(),
+        streaming_response=iter_sse_lines(response.iter_text()),
         sync_stream=True,
         json_mode=json_mode,
         speed=speed,

litellm/llms/anthropic/messages/caller_model_checks.py

@@ -268,6 +268,7 @@ async def caller_within_model_budget(
                 user_id=user_id,
                 user_model_max_budget=user_model_max_budget,
                 model=model,
+                team_id=user_api_key_auth.team_id,
             )
         except litellm.BudgetExceededError:
             return "denied"
@@ -309,6 +310,7 @@ async def caller_within_model_budget(
                 end_user_id=end_user_id,
                 end_user_model_max_budget=end_user_model_max_budget,
                 model=model,
+                team_id=user_api_key_auth.team_id,
             )
         except litellm.BudgetExceededError:
             return "denied"

litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py

@@ -2,15 +2,16 @@
 ## Translates OpenAI call to Anthropic `/v1/messages` format
 import asyncio
 from collections import deque
-from collections.abc import AsyncGenerator, Iterator, Mapping
+from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final
 
+import anyio
 from pydantic import ConfigDict, field_validator
 
 from litellm import verbose_logger
 from litellm._logging import redact_internal_details_from_client_message
 from litellm._uuid import uuid
-from litellm.exceptions import MidStreamFallbackError
+from litellm.exceptions import APIError, MidStreamFallbackError
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     encrypted_reasoning_signature,
 )
@@ -22,6 +23,7 @@ from litellm.llms.anthropic.messages.native.utils import (
 from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
 from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
 from litellm.responses.streaming_iterator import (
+    map_stream_error_to_exception,
     partial_responses_incomplete_event_on_disconnect,
     stream_error_status_and_message,
 )
@@ -130,10 +132,12 @@ class AnthropicResponsesStreamWrapper:
     ) -> None:
         self.responses_stream = responses_stream
         self.model = model
+        self.litellm_logging_obj = litellm_logging_obj
         self.safeguards_evaluator: Final = safeguards_evaluator
         self._message_id: str = f"msg_{uuid.uuid4()}"
         if litellm_logging_obj is not None:
             litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
+        self._unreported_failure: Exception | None = None
         self._current_block_index: int = -1
         # Map item_id -> content_block_index so we can stop the right block later
         self._item_id_to_block_index: dict[str, int] = {}
@@ -377,16 +381,24 @@ class AnthropicResponsesStreamWrapper:
 
         if event_type == "response.failed":
             failed: Final = _FailedResponseEvent.model_validate(event)
-            status_code, message = stream_error_status_and_message(
-                failed.response.error if failed.response is not None else None
-            )
+            upstream_error: Final = failed.response.error if failed.response is not None else None
+            status_code, message = stream_error_status_and_message(upstream_error)
             verbose_logger.error(
                 "AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed (%s): %s",
                 self.model,
                 status_code,
                 message,
             )
-            self._fail_stream(status_code, message)
+            self._fail_stream(
+                status_code,
+                message,
+                failure=map_stream_error_to_exception(
+                    upstream_error,
+                    self.model,
+                    getattr(self.responses_stream, "custom_llm_provider", None) or "",
+                    logging_obj=self.litellm_logging_obj,
+                ),
+            )
             return
 
         # ---- response completed -> message_delta + message_stop ----
@@ -439,10 +451,36 @@ class AnthropicResponsesStreamWrapper:
             self._sent_message_stop = True
             return
 
-    def _fail_stream(self, status_code: int, message: str) -> None:
+    def _fail_stream(self, status_code: int, message: str, failure: Exception | None = None) -> None:
         self._stream_failed = True
+        self._unreported_failure = failure or APIError(
+            status_code=status_code,
+            message=message,
+            llm_provider=getattr(self.responses_stream, "custom_llm_provider", None),
+            model=self.model,
+        )
         self._chunk_queue.append(_anthropic_error_chunk(status_code, message))
 
+    async def _report_unreported_failure(self) -> None:
+        failure: Final = self._unreported_failure
+        if failure is None:
+            return
+        self._unreported_failure = None
+        on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr(
+            self.litellm_logging_obj, "_on_detached_stream_failure", None
+        )
+        if on_failure is None:
+            return
+        try:
+            with anyio.CancelScope(shield=True):
+                await on_failure(failure)
+        except Exception as hook_failure:  # noqa: BLE001  # a failing proxy hook must not hide the error event from the client
+            verbose_logger.warning(
+                "AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)",
+                type(hook_failure).__name__,
+                hook_failure,
+            )
+
     def __aiter__(self) -> "AnthropicResponsesStreamWrapper":
         return self
 
@@ -450,6 +488,7 @@ class AnthropicResponsesStreamWrapper:
         if self._chunk_queue:
             return self._chunk_queue.popleft()
         if self._stream_failed:
+            await self._report_unreported_failure()
             raise StopAsyncIteration
 
         if not self._sent_message_start:
@@ -478,7 +517,7 @@ class AnthropicResponsesStreamWrapper:
             verbose_logger.exception(
                 "AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed", self.model
             )
-            self._fail_stream(*_failure_status_and_message(e))
+            self._fail_stream(*_failure_status_and_message(e), failure=_original_failure(e))
 
         if not self._chunk_queue and not self._sent_message_stop and not self._stream_failed:
             verbose_logger.error(
@@ -498,8 +537,14 @@ class AnthropicResponsesStreamWrapper:
 
     async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
         safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
-        async for chunk in self:
-            yield anthropic_sse_frame(await safeguard_results.observe(chunk))
+        try:
+            async for chunk in self:
+                yield anthropic_sse_frame(await safeguard_results.observe(chunk))
+        finally:
+            from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt
+
+            if not closing_superseded_attempt():
+                await self._report_unreported_failure()
 
     def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
         return partial_responses_incomplete_event_on_disconnect(self.responses_stream)

litellm/llms/anthropic/messages/stream_assembly.py

@@ -13,6 +13,7 @@ from litellm.llms.anthropic.chat.handler import (
 )
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS
+from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.types.utils import (
     Choices,
     Message,
@@ -269,7 +270,7 @@ def build_complete_streaming_response_legacy(
 
 def extract_sse_data(event_str: str) -> dict | None:
     """Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
-    for line in event_str.splitlines():
+    for line in split_sse_lines(event_str):
         stripped = line.strip()
         if stripped.startswith("data:"):
             payload = stripped[len("data:") :].strip()

litellm/llms/anthropic/prompt_cache_prediction.py

@@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
     AnthropicMessagesConfig,
 )
 from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
-from litellm.router_utils.baseline_request import (
+from litellm.router_strategy.auto_router.baseline_request import (
     BASELINE_PARAMETERS,
     capture_baseline_parameters,
 )

litellm/llms/azure/azure.py

@@ -13,13 +13,12 @@ from openai import (
     BadRequestError,
     OpenAI,
 )
-from pydantic import BaseModel, TypeAdapter
+from pydantic import BaseModel
 
 import litellm
-from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
+from litellm.constants import DEFAULT_MAX_RETRIES
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
-from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -34,7 +33,6 @@ from litellm.types.utils import (
 from litellm.utils import (
     CustomStreamWrapper,
     convert_to_model_response_object,
-    modify_url,
 )
 
 from ...types.llms.openai import HttpxBinaryResponseContent
@@ -59,8 +57,6 @@ from .image_generation import (
 )
 from .image_generation.http_utils import azure_deployment_image_generation_json_body
 
-_DECODED_JSON: Final = TypeAdapter(object)
-
 
 class AzureOpenAIAssistantsAPIConfig:
     """
@@ -860,11 +856,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         headers: dict,
         deployment_name: str | None = None,
     ) -> httpx.Response:
-        """
-        Implemented for azure dall-e-2 image gen calls
-
-        Alternative to needing a custom transport implementation
-        """
         if client is None:
             _params: Final = {}
             if timeout is not None:
@@ -881,95 +872,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         else:
             async_handler = client
 
-        if (
-            "images/generations" in api_base
-            and api_version
-            in [  # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
-                "2023-06-01-preview",
-                "2023-07-01-preview",
-                "2023-08-01-preview",
-                "2023-09-01-preview",
-                "2023-10-01-preview",
-            ]
-        ):  # CREATE + POLL for azure dall-e-2 calls
-            api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
-
-            data.pop(
-                "model", None
-            )  # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
-            response = await async_handler.post(
-                url=api_base,
-                data=json.dumps(data),
-                headers=headers,
-            )
-            if "operation-location" in response.headers:
-                operation_location_url: Final = response.headers["operation-location"]
-            else:
-                raise AzureOpenAIError(status_code=500, message=response.text)
-            # Reject polling URLs that don't share an origin with ``api_base``.
-            # Without this an upstream-controlled or attacker-controlled
-            # value would receive the operator's Azure API key in the
-            # request headers below. VERIA-51.
-            try:
-                assert_same_origin(operation_location_url, api_base)
-            except SSRFError as ssrf_err:
-                raise AzureOpenAIError(
-                    status_code=502,
-                    message=f"Rejected polling URL: {ssrf_err}",
-                )
-            response = await async_handler.get(
-                url=operation_location_url,
-                headers=headers,
-            )
-
-            await response.aread()
-
-            timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
-            start_time: Final = time.time()
-            if "status" not in response.json():
-                # Don't reflect the raw response body — when the polling
-                # URL points at an internal JSON API (cloud metadata
-                # service etc.) reflecting it here turns Blind SSRF into
-                # Full-Read SSRF. VERIA-51.
-                raise AzureOpenAIError(
-                    status_code=502,
-                    message="Polling response missing 'status' field",
-                )
-            while response.json()["status"] not in ["succeeded", "failed"]:
-                if time.time() - start_time > timeout_secs:
-                    raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
-
-                await asyncio.sleep(int(response.headers.get("retry-after") or 10))
-                response = await async_handler.get(
-                    url=operation_location_url,
-                    headers=headers,
-                )
-                await response.aread()
-
-            if response.json()["status"] == "failed":
-                error_data: Final = response.json()
-                # Preserve Azure error details (e.g. content_policy_violation,
-                # inner_error, content_filter_results) as structured body so
-                # exception_type() can route them correctly.
-                _error_body: Final = error_data.get("error", error_data)
-                _error_msg: Final = (
-                    _error_body.get("message", "Image generation failed")
-                    if isinstance(_error_body, dict)
-                    else json.dumps(error_data)
-                )
-                raise AzureOpenAIError(
-                    status_code=400,
-                    message=_error_msg,
-                    body=error_data,
-                )
-
-            result: Final = _DECODED_JSON.validate_python(response.json()["result"])
-            return httpx.Response(
-                status_code=200,
-                headers=response.headers,
-                content=json.dumps(result).encode("utf-8"),
-                request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
-            )
         request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
         return await async_handler.post(
             url=api_base,
@@ -988,11 +890,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         headers: dict,
         deployment_name: str | None = None,
     ) -> httpx.Response:
-        """
-        Implemented for azure dall-e-2 image gen calls
-
-        Alternative to needing a custom transport implementation
-        """
         if client is None:
             _params: Final = {}
             if timeout is not None:
@@ -1006,87 +903,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         else:
             sync_handler = client
 
-        if (
-            "images/generations" in api_base
-            and api_version
-            in [  # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
-                "2023-06-01-preview",
-                "2023-07-01-preview",
-                "2023-08-01-preview",
-                "2023-09-01-preview",
-                "2023-10-01-preview",
-            ]
-        ):  # CREATE + POLL for azure dall-e-2 calls
-            api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
-
-            data.pop(
-                "model", None
-            )  # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
-            response = sync_handler.post(
-                url=api_base,
-                data=json.dumps(data),
-                headers=headers,
-            )
-            if "operation-location" in response.headers:
-                operation_location_url: Final = response.headers["operation-location"]
-            else:
-                raise AzureOpenAIError(status_code=500, message=response.text)
-            try:
-                assert_same_origin(operation_location_url, api_base)
-            except SSRFError as ssrf_err:
-                raise AzureOpenAIError(
-                    status_code=502,
-                    message=f"Rejected polling URL: {ssrf_err}",
-                )
-            response = sync_handler.get(
-                url=operation_location_url,
-                headers=headers,
-            )
-
-            response.read()
-
-            timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
-            start_time: Final = time.time()
-            if "status" not in response.json():
-                raise AzureOpenAIError(
-                    status_code=502,
-                    message="Polling response missing 'status' field",
-                )
-            while response.json()["status"] not in ["succeeded", "failed"]:
-                if time.time() - start_time > timeout_secs:
-                    raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
-
-                time.sleep(int(response.headers.get("retry-after") or 10))
-                response = sync_handler.get(
-                    url=operation_location_url,
-                    headers=headers,
-                )
-                response.read()
-
-            if response.json()["status"] == "failed":
-                error_data: Final = response.json()
-                # Preserve Azure error details (e.g. content_policy_violation,
-                # inner_error, content_filter_results) as structured body so
-                # exception_type() can route them correctly.
-                _error_body: Final = error_data.get("error", error_data)
-                _error_msg: Final = (
-                    _error_body.get("message", "Image generation failed")
-                    if isinstance(_error_body, dict)
-                    else json.dumps(error_data)
-                )
-                raise AzureOpenAIError(
-                    status_code=400,
-                    message=_error_msg,
-                    body=error_data,
-                )
-
-            result: Final = _DECODED_JSON.validate_python(response.json()["result"])
-            return httpx.Response(
-                status_code=200,
-                headers=response.headers,
-                content=json.dumps(result).encode("utf-8"),
-                request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
-            )
         request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
         return sync_handler.post(
             url=api_base,
@@ -1193,10 +1009,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 deployment_name=model,
             )
 
-            provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
+            provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
             if provider_config is not None:
                 return provider_config.transform_image_generation_response(
-                    model=data.get("model", "dall-e-2"),
+                    model=data.get("model", ""),
                     raw_response=httpx_response,
                     model_response=model_response or ImageResponse(),
                     logging_obj=logging_obj,
@@ -1329,10 +1145,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 headers=request_headers,
                 deployment_name=model,
             )
-            provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
+            provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
             if isinstance(provider_config, AzureFoundryMAIImageGenerationConfig):
                 return provider_config.transform_image_generation_response(
-                    model=data.get("model", "dall-e-2"),
+                    model=data.get("model", ""),
                     raw_response=httpx_response,
                     model_response=model_response or ImageResponse(),
                     logging_obj=logging_obj,

litellm/llms/azure/image_generation/__init__.py

@@ -4,14 +4,10 @@ from litellm.llms.base_llm.image_generation.transformation import (
     BaseImageGenerationConfig,
 )
 
-from .dall_e_2_transformation import AzureDallE2ImageGenerationConfig
-from .dall_e_3_transformation import AzureDallE3ImageGenerationConfig
 from .gpt_transformation import AzureGPTImageGenerationConfig
 from .http_utils import azure_deployment_image_generation_json_body
 
 __all__ = [
-    "AzureDallE2ImageGenerationConfig",
-    "AzureDallE3ImageGenerationConfig",
     "AzureGPTImageGenerationConfig",
     "azure_deployment_image_generation_json_body",
 ]
@@ -21,11 +17,7 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
     model = model.lower()
     model = model.replace("-", "")
     model = model.replace("_", "")
-    if model == "" or "dalle2" in model:  # empty model is dall-e-2
-        return AzureDallE2ImageGenerationConfig()
-    elif "dalle3" in model:
-        return AzureDallE3ImageGenerationConfig()
-    elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
+    if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
         return AzureFoundryMAIImageGenerationConfig()
     else:
         verbose_logger.debug(

litellm/llms/azure/image_generation/dall_e_2_transformation.py

@@ -1,7 +0,0 @@
-from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
-
-
-class AzureDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
-    """
-    Azure dall-e-2 image generation config
-    """

litellm/llms/azure/image_generation/dall_e_3_transformation.py

@@ -1,7 +0,0 @@
-from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
-
-
-class AzureDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
-    """
-    Azure dall-e-3 image generation config
-    """

litellm/llms/azure_ai/agents/handler.py

@@ -35,6 +35,7 @@ from litellm.llms.azure_ai.agents.transformation import (
     AzureAIAgentsConfig,
     AzureAIAgentsError,
 )
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
 from litellm.types.llms.openai import (
     ChatCompletionAnnotation,
     ChatCompletionAnnotationURLCitation,
@@ -632,7 +633,7 @@ class AzureAIAgentsHandler:
 
         current_event = None
 
-        async for line in response.aiter_lines():
+        async for line in aiter_sse_lines(response.aiter_text()):
             line = line.strip()
 
             if line.startswith("event:"):

litellm/llms/azure_ai/image_generation/__init__.py

@@ -3,15 +3,11 @@ from litellm.llms.base_llm.image_generation.transformation import (
     BaseImageGenerationConfig,
 )
 
-from .dall_e_2_transformation import AzureFoundryDallE2ImageGenerationConfig
-from .dall_e_3_transformation import AzureFoundryDallE3ImageGenerationConfig
 from .flux_transformation import AzureFoundryFluxImageGenerationConfig
 from .gpt_transformation import AzureFoundryGPTImageGenerationConfig
 from .mai_transformation import AzureFoundryMAIImageGenerationConfig
 
 __all__ = [
-    "AzureFoundryDallE2ImageGenerationConfig",
-    "AzureFoundryDallE3ImageGenerationConfig",
     "AzureFoundryFluxImageGenerationConfig",
     "AzureFoundryGPTImageGenerationConfig",
     "AzureFoundryMAIImageGenerationConfig",
@@ -22,11 +18,7 @@ def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfi
     model = model.lower()
     model = model.replace("-", "")
     model = model.replace("_", "")
-    if model == "" or "dalle2" in model:  # empty model is dall-e-2
-        return AzureFoundryDallE2ImageGenerationConfig()
-    elif "dalle3" in model:
-        return AzureFoundryDallE3ImageGenerationConfig()
-    elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
+    if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
         return AzureFoundryMAIImageGenerationConfig()
     elif "flux" in model:
         return AzureFoundryFluxImageGenerationConfig()

litellm/llms/azure_ai/image_generation/dall_e_2_transformation.py

@@ -1,7 +0,0 @@
-from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
-
-
-class AzureFoundryDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
-    """
-    Azure dall-e-2 image generation config
-    """

litellm/llms/azure_ai/image_generation/dall_e_3_transformation.py

@@ -1,7 +0,0 @@
-from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
-
-
-class AzureFoundryDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
-    """
-    Azure dall-e-3 image generation config
-    """

litellm/llms/base_llm/agents/transformation.py

@@ -15,6 +15,7 @@ from collections.abc import Mapping
 import httpx
 
 from litellm.types.agents import (
+    AgentCreateRequest,
     AgentCreateResponse,
     AgentDeleteResult,
     AgentListResponse,
@@ -50,10 +51,9 @@ class BaseAgentsAPIConfig(ABC):
     @abstractmethod
     def transform_create_request(
         self,
-        name: str,
-        litellm_params: Mapping[str, object],
-    ) -> dict[str, object]:
-        """Map name + litellm_params to the provider's create-agent body."""
+        create_request: AgentCreateRequest,
+    ) -> Mapping[str, object]:
+        """Map the create request to the provider's create-agent body."""
 
     @abstractmethod
     def transform_create_response(

litellm/llms/base_llm/base_model_iterator.py

@@ -96,7 +96,7 @@ class BaseModelResponseIterator:
         released (and a backend like vLLM aborts generation) when the stream
         is abandoned before its natural end.
 
-        ``streaming_response`` is usually a bare ``aiter_lines()`` generator
+        ``streaming_response`` is usually a bare ``aiter_sse_lines()`` generator
         that holds no reference to the response, so the handler that owns the
         response attaches it here after construction."""
         if self.http_response is not None:

litellm/llms/base_llm/files/transformation.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Union
+from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Final, Union
 
 import httpx
 from openai.types.file_deleted import FileDeleted
@@ -38,6 +39,22 @@ else:
     UserAPIKeyAuth = Any
 
 
+def _routes_nowhere(_model_name: str) -> bool:
+    return False
+
+
+@dataclass(frozen=True, slots=True)
+class FileListModelScope:
+    model_mapping_keys: frozenset[str]
+    routes_stored_model_name: Callable[[str], bool] = _routes_nowhere
+
+    def holds(self, model_mapping_keys: Iterable[str], stored: bool) -> bool:
+        mapped_keys: Final = frozenset(model_mapping_keys)
+        return not self.model_mapping_keys.isdisjoint(mapped_keys) or (
+            stored and any(map(self.routes_stored_model_name, mapped_keys))
+        )
+
+
 class BaseFileUploadStream(ABC):
     """Re-iterable request body that yields an upload's bytes lazily.
 
@@ -274,6 +291,8 @@ class BaseFileEndpoints(ABC):
         user_api_key_dict: UserAPIKeyAuth,
         limit: int | None = None,
         after: str | None = None,
+        model_scope: FileListModelScope | None = None,
+        before: str | None = None,
         **data: dict,
     ) -> FileListPage:
         pass

litellm/llms/base_llm/guardrail_translation/utils.py

@@ -6,6 +6,7 @@ from typing import Final, TypeVar, cast  # noqa: TID251  # a rebuilt chat row ha
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
+from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
 from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
 
@@ -24,7 +25,7 @@ def _anthropic_stream_chunk_events(item: object) -> list[dict]:
 
     events: Final[list[dict]] = []
     for block in chunk.split("\n\n"):
-        for line in block.splitlines():
+        for line in split_sse_lines(block):
             stripped = line.strip()
             if not stripped.startswith("data:"):
                 continue

litellm/llms/base_llm/sandbox/transformation.py

@@ -12,6 +12,7 @@ from typing import Final
 import httpx
 from pydantic import Field, PrivateAttr
 
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
 from litellm.types.llms.base import LiteLLMPydanticObjectBase
 
 SANDBOX_MAX_OUTPUT_BYTES: Final = 10 * 1024 * 1024
@@ -99,7 +100,7 @@ class BaseSandboxConfig:
     async def _read_capped_lines(self, response: httpx.Response) -> list[str]:
         lines: Final[list[str]] = []
         total = 0
-        async for line in response.aiter_lines():
+        async for line in aiter_sse_lines(response.aiter_text()):
             total += len(line.encode("utf-8"))
             if total > SANDBOX_MAX_OUTPUT_BYTES:
                 raise ValueError(

litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py

@@ -10,6 +10,8 @@ import time
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -34,6 +36,15 @@ else:
     LiteLLMLoggingObj = Any
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _PegasusResponse(TypedDict):
+    message: ReadOnly[NotRequired[str | None]]
+    finishReason: ReadOnly[NotRequired[str | None]]
+
+
+_PEGASUS_RESPONSE: Final = TypeAdapter(_PegasusResponse)
+
+
 class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
     """
     Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
@@ -207,7 +218,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
         ModelResponse with choices[0].message.content and finish_reason
         """
         try:
-            completion_response: Final = raw_response.json()
+            completion_response: Final = _PEGASUS_RESPONSE.validate_python(raw_response.json())
         except Exception as e:
             raise BedrockError(
                 message=f"Error parsing response: {raw_response.text}, error: {e}",
@@ -224,7 +235,7 @@ class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
         message_content: Final = completion_response.get("message", "")
 
         # Extract finish reason and map to LiteLLM format
-        finish_reason_raw: Final = completion_response.get("finishReason", "stop")
+        finish_reason_raw: Final = completion_response.get("finishReason") or "stop"
         finish_reason: Final = map_finish_reason(finish_reason_raw)
 
         # Set the response content

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -1,11 +1,12 @@
 import copy
 import json
 import time
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, cast, get_args
 
 import httpx
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -48,6 +49,27 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 
 _GUARDRAIL_CONFIG_VALIDATOR: Final["TypeAdapter[GuardrailConfigBlock]"] = TypeAdapter(GuardrailConfigBlock)
 
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CohereGeneration(TypedDict):
+    text: ReadOnly[str | None]
+    finish_reason: ReadOnly[str | None]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CohereInvokeResponse(TypedDict):
+    text: NotRequired[ReadOnly[str | None]]
+    generations: NotRequired[ReadOnly[Sequence[_CohereGeneration]]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _LlamaInvokeResponse(TypedDict):
+    generation: ReadOnly[str | None]
+
+
+_COHERE_INVOKE_RESPONSE: Final = TypeAdapter(_CohereInvokeResponse)
+_LLAMA_INVOKE_RESPONSE: Final = TypeAdapter(_LlamaInvokeResponse)
+
 _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
     "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
 )
@@ -310,12 +332,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         outputText: str | None = None
         try:
             if provider == "cohere":
-                if "text" in completion_response:
-                    outputText = completion_response["text"]
-                elif "generations" in completion_response:
-                    outputText = completion_response["generations"][0]["text"]
+                cohere_response: Final = _COHERE_INVOKE_RESPONSE.validate_python(completion_response)
+                if "text" in cohere_response:
+                    outputText = cohere_response["text"]
+                elif "generations" in cohere_response:
+                    outputText = cohere_response["generations"][0]["text"]
                     model_response.choices[0].finish_reason = map_finish_reason(
-                        completion_response["generations"][0]["finish_reason"]
+                        cohere_response["generations"][0]["finish_reason"] or "stop"
                     )
             elif provider == "anthropic":
                 return litellm.AmazonAnthropicClaudeConfig().transform_response(
@@ -361,7 +384,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
             elif provider == "ai21":
                 outputText = completion_response.get("completions")[0].get("data").get("text")
             elif provider == "meta" or provider == "llama" or provider == "deepseek_r1":
-                outputText = completion_response["generation"]
+                outputText = _LLAMA_INVOKE_RESPONSE.validate_python(completion_response)["generation"]
             elif provider == "mistral":
                 outputText = litellm.AmazonMistralConfig.get_outputText(completion_response, model_response)
             else:  # amazon titan

litellm/llms/bedrock/files/transformation.py

@@ -49,6 +49,7 @@ from litellm.llms.base_llm.files.transformation import (
     BaseFilesConfig,
     LiteLLMLoggingObj,
 )
+from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.llms.bedrock import AwsAuthParams, BedrockBatchRecordKind
 from litellm.types.llms.openai import (
@@ -400,7 +401,7 @@ def _batch_input_line(line_number: int, line: str) -> _BatchInputLine:
 
 
 def _non_blank_lines(file_content: str) -> Iterator[tuple[int, str]]:
-    numbered_lines: Final = enumerate(file_content.splitlines(), start=1)
+    numbered_lines: Final = enumerate(split_sse_lines(file_content), start=1)
     return ((line_number, line) for line_number, line in numbered_lines if line.strip())
 
 

litellm/llms/bedrock/passthrough/guardrail_translation/handler.py

@@ -45,6 +45,7 @@ from litellm.llms.bedrock.passthrough.guardrail_translation.structured_edits imp
     structured_edits,
     tool_edits,
 )
+from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.proxy.guardrails.auto_router_compression import is_compression_guardrail
 from litellm.types.utils import GenericGuardrailAPIInputs
 
@@ -947,7 +948,7 @@ def _invoke_frames(items: Sequence[object]) -> tuple[InvokeStreamFrame, ...]:
 def _sse_events(sse: bytes) -> tuple[Mapping[str, object], ...]:
     return tuple(
         _JSON_OBJECT.validate_python(json.loads(line[len("data:") :]))
-        for line in sse.decode().splitlines()
+        for line in split_sse_lines(sse.decode())
         if line.startswith("data:")
     )
 

litellm/llms/bedrock/realtime/transformation.py

@@ -53,6 +53,7 @@ from litellm.types.realtime import (
     RealtimeResponseTypedDict,
 )
 
+_JSON_OBJECT_MESSAGE: Final = TypeAdapter(dict[str, JsonValue], config=ConfigDict(strict=True))
 _JSON_VALUE: Final = TypeAdapter(object)
 _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
 
@@ -856,7 +857,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             List of Bedrock format messages (JSON strings)
         """
         try:
-            json_message: Final[dict[str, JsonValue]] = json.loads(message)
+            json_message: Final = _JSON_OBJECT_MESSAGE.validate_python(json.loads(message))
         except json.JSONDecodeError:
             verbose_logger.warning("Invalid JSON message: %s", message[:200])
             return []

litellm/llms/bedrock/search/transformation.py

@@ -47,6 +47,7 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import BedrockError
+from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.secret_managers.main import get_secret_str
 
 # AgentCore web-search rejects queries longer than 200 characters
@@ -146,7 +147,9 @@ def _iter_sse_events(text: str) -> Iterator[Mapping[str, object]]:
     progress notifications before the JSON-RPC response.
     """
     for chunk in _SSE_EVENT_SEPARATOR.split(text):
-        payload = "\n".join(line[len("data:") :].lstrip() for line in chunk.splitlines() if line.startswith("data:"))
+        payload = "\n".join(
+            line[len("data:") :].lstrip() for line in split_sse_lines(chunk) if line.startswith("data:")
+        )
         if not payload:
             continue
         try:

litellm/llms/chatgpt/responses/transformation.py

@@ -11,6 +11,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
     safe_convert_created_field,
 )
 from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.custom_httpx.sse_lines import split_sse_lines
 from litellm.llms.openai.common_utils import OpenAIError
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.responses.sse_output_recovery import (
@@ -199,7 +200,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
         error_message = None
         streamed_output_items: Final[dict[int, dict[str, object]]] = {}
         text_only_output_items: Final[dict[int, dict[str, object]]] = {}
-        for chunk in body_text.splitlines():
+        for chunk in split_sse_lines(body_text):
             parsed_chunk = parse_sse_json_chunk(chunk)
             if parsed_chunk is None:
                 continue

litellm/llms/codestral/completion/handler.py

@@ -20,6 +20,7 @@ from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     get_async_httpx_client,
 )
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
 from litellm.types.utils import TextChoices
 from litellm.utils import CustomStreamWrapper, TextCompletionResponse
 
@@ -109,7 +110,7 @@ async def make_call(
     if response.status_code != 200:
         raise TextCompletionCodestralError(status_code=response.status_code, message=response.text)
 
-    completion_stream: Final = response.aiter_lines()
+    completion_stream: Final = aiter_sse_lines(response.aiter_text())
     # LOGGING
     logging_obj.post_call(
         input=messages,
@@ -339,13 +340,13 @@ class CodestralTextCompletion:
             response = litellm.module_level_client.post(
                 completion_url,
                 headers=headers,
-                data=json.dumps(data),
+                data=json.dumps({**data, "stream": True}),
                 stream=stream,
             )
             _response: Final = CustomStreamWrapper(
-                response.iter_lines(),
+                iter_sse_lines(response.iter_text()),
                 model,
-                custom_llm_provider="codestral",
+                custom_llm_provider="text-completion-codestral",
                 logging_obj=logging_obj,
             )
             return _response

litellm/llms/custom_httpx/aiohttp_handler.py

@@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import (
     get_httpx_client,
     get_ssl_configuration,
 )
+from litellm.llms.custom_httpx.sse_lines import iter_sse_lines
 from litellm.types.llms.openai import FileTypes
 from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders
 from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
@@ -466,7 +467,7 @@ class BaseLLMAIOHTTPHandler:
             )
         else:
             completion_stream = provider_config.get_model_response_iterator(
-                streaming_response=response.iter_lines(), sync_stream=True
+                streaming_response=iter_sse_lines(response.iter_text()), sync_stream=True
             )
 
         # LOGGING
@@ -560,11 +561,8 @@ class BaseLLMAIOHTTPHandler:
         organization: str | None = None,
         headers: dict | None = None,
     ) -> ImageResponse:
-        if model is None:
-            raise ValueError("model is required for non-openai image variations")
-
         provider_config: Final = ProviderConfigManager.get_provider_image_variation_config(
-            model=model,  # openai defaults to dall-e-2
+            model=model or "",
             provider=LlmProviders(custom_llm_provider),
         )
 

litellm/llms/custom_httpx/aiohttp_transport.py

@@ -6,6 +6,7 @@ import ssl
 import sys
 import typing
 import urllib.request
+import weakref
 from collections.abc import Callable, Generator
 from typing import ClassVar, Final
 
@@ -109,6 +110,24 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
         raise mapped_exc(message) from exc
 
 
+def _release_transport_anchor(_transport: object) -> None:
+    """Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
+
+
+def _anchor_transport_to(stream: "AiohttpResponseStream", transport: object) -> None:
+    """Keep the transport alive for as long as the stream can still read.
+
+    The transport finalizer closes the session it owns, and nothing on the path from a
+    live response back to that session references the transport, so a client dropped
+    mid-stream would close the session under the body. ``weakref.finalize`` holds the
+    transport in its own registry rather than on the stream, which keeps the transport
+    out of the response's reference cycle (an exception carrying the response, for one)
+    so it is finalized by refcount once the anchor drops and can still schedule an async
+    close, instead of being collected in the same cyclic pass as the session it closes.
+    """
+    weakref.finalize(stream, _release_transport_anchor, transport)
+
+
 class AiohttpResponseStream(httpx.AsyncByteStream):
     CHUNK_SIZE = 1024 * 16
 
@@ -463,10 +482,12 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
                 # Re-raise if it's a different RuntimeError
                 raise
 
+        stream: Final = AiohttpResponseStream(response)
+        _anchor_transport_to(stream, self)
         return httpx.Response(
             status_code=response.status,
             headers=response.headers,
-            stream=AiohttpResponseStream(response),
+            stream=stream,
             request=request,
         )
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -118,6 +118,7 @@ from litellm.llms.custom_httpx.http_handler import (
     get_httpx_client,
     header_value,
 )
+from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines, iter_sse_lines
 from litellm.responses.streaming_iterator import (
     BaseResponsesAPIStreamingIterator,
     MockResponsesAPIStreamingIterator,
@@ -1086,7 +1087,7 @@ class BaseLLMHTTPHandler:
             completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
         else:
             completion_stream = provider_config.get_model_response_iterator_for_request(
-                streaming_response=response.iter_lines(),
+                streaming_response=iter_sse_lines(response.iter_text()),
                 sync_stream=True,
                 json_mode=json_mode,
                 optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),
@@ -1229,7 +1230,7 @@ class BaseLLMHTTPHandler:
             completion_stream: object = MockResponseIterator(model_response=model_response, json_mode=json_mode)
         else:
             completion_stream = provider_config.get_model_response_iterator_for_request(
-                streaming_response=response.aiter_lines(),
+                streaming_response=aiter_sse_lines(response.aiter_text()),
                 sync_stream=False,
                 json_mode=json_mode,
                 optional_params=_OPTIONAL_PARAMS.validate_python(optional_params),

litellm/llms/custom_httpx/sse_lines.py

@@ -0,0 +1,74 @@
+"""
+Split a decoded Server-Sent Events body into lines the way the SSE spec does.
+
+httpx's `iter_lines()` / `aiter_lines()` follow `str.splitlines()`, which also breaks on U+0085, U+2028,
+U+2029, \\x0b, \\x0c and \\x1c-\\x1e. JSON leaves U+0085, U+2028 and U+2029 unescaped inside strings, so a
+`data:` line carrying one of them gets cut in half. SSE lines end only at CRLF, LF or CR
+(https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream)
+"""
+
+import io
+import re
+from collections.abc import AsyncIterable, AsyncIterator, Iterable, Iterator, Sequence
+from typing import Final
+
+_SSE_LINE_BREAK: Final = re.compile(r"\r\n|\r|\n")
+_SSE_LINE_WITH_BREAK: Final = re.compile(r"[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+")
+
+
+def _split_lines(text: str) -> Sequence[str]:
+    if "\r" not in text:
+        return text.split("\n")
+    crlf_folded: Final = text.replace("\r\n", "\n")
+    if "\r" not in crlf_folded:
+        return crlf_folded.split("\n")
+    return _SSE_LINE_BREAK.split(text)
+
+
+class _SSELineSplitter:
+    def __init__(self) -> None:
+        self._partial_line: io.StringIO = io.StringIO()
+        self._after_cr: bool = False
+
+    def feed(self, text: str) -> tuple[str, ...]:
+        if not text:
+            return ()
+        fresh: Final = text[1:] if self._after_cr and text.startswith("\n") else text
+        self._after_cr = fresh.endswith("\r")
+        *ended, tail = _split_lines(fresh)
+        if not ended:
+            _ = self._partial_line.write(tail)
+            return ()
+        first: Final = self._partial_line.getvalue() + ended[0]
+        self._partial_line = io.StringIO()
+        _ = self._partial_line.write(tail)
+        return (first, *ended[1:])
+
+    def flush(self) -> tuple[str, ...]:
+        rest: Final = self._partial_line.getvalue()
+        self._partial_line = io.StringIO()
+        return (rest,) if rest else ()
+
+
+def iter_sse_lines(text_chunks: Iterable[str]) -> Iterator[str]:
+    splitter: Final = _SSELineSplitter()
+    for text in text_chunks:
+        yield from splitter.feed(text)
+    yield from splitter.flush()
+
+
+def split_sse_lines(text: str) -> tuple[str, ...]:
+    return tuple(iter_sse_lines((text,)))
+
+
+def split_sse_lines_keepends(text: str) -> tuple[str, ...]:
+    return tuple(_SSE_LINE_WITH_BREAK.findall(text))
+
+
+async def aiter_sse_lines(text_chunks: AsyncIterable[str]) -> AsyncIterator[str]:
+    splitter: Final = _SSELineSplitter()
+    async for text in text_chunks:
+        for line in splitter.feed(text):
+            yield line
+    for line in splitter.flush():
+        yield line

litellm/llms/dashscope/embed/transformation.py

@@ -14,6 +14,8 @@ Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, with_config
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import ensure_api_route
@@ -28,6 +30,23 @@ from ..common_utils import DashScopeError
 DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _DashScopeUsage(TypedDict, total=False):
+    prompt_tokens: NotRequired[ReadOnly[int]]
+    total_tokens: NotRequired[ReadOnly[int]]
+
+
+@with_config(ConfigDict(extra="allow", strict=True))
+class _DashScopeEmbeddingResponse(TypedDict, total=False):
+    data: NotRequired[ReadOnly[list[object]]]
+    model: NotRequired[ReadOnly[str | None]]
+    usage: NotRequired[ReadOnly[_DashScopeUsage | None]]
+    id: NotRequired[ReadOnly[object]]
+
+
+_DASHSCOPE_EMBEDDING_RESPONSE: Final = TypeAdapter(_DashScopeEmbeddingResponse)
+
+
 class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
     """
     Reference: https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
@@ -152,11 +171,12 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
                 message=message,
             )
 
+        parsed_response: Final = _DASHSCOPE_EMBEDDING_RESPONSE.validate_python(response_json)
         model_response.object = "list"
-        model_response.data = response_json.get("data", [])
-        model_response.model = response_json.get("model", model)
+        model_response.data = parsed_response.get("data", [])
+        model_response.model = parsed_response.get("model", model)
 
-        usage: Final = response_json.get("usage") or {}
+        usage: Final = parsed_response.get("usage") or {}
         prompt_tokens: Final = usage.get("prompt_tokens", 0)
         total_tokens: Final = usage.get("total_tokens", prompt_tokens)
         setattr(
@@ -169,8 +189,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
             ),
         )
 
-        if "id" in response_json:
-            setattr(model_response, "id", response_json["id"])
+        if "id" in parsed_response:
+            setattr(model_response, "id", parsed_response["id"])
 
         return model_response
 

litellm/llms/databricks/streaming_utils.py

@@ -1,6 +1,8 @@
 import json
 from typing import Final
 
+from pydantic import ConfigDict, TypeAdapter
+
 import litellm
 from litellm import verbose_logger
 from litellm.types.llms.openai import (
@@ -23,6 +25,9 @@ def _usage_block(usage: Usage) -> ChatCompletionUsageBlock:
     )
 
 
+_JSON_CHUNK: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True))
+
+
 class ModelResponseIterator:
     def __init__(self, streaming_response, sync_stream: bool):
         self.streaming_response = streaming_response
@@ -99,7 +104,7 @@ class ModelResponseIterator:
             self.response_iterator = self.streaming_response
         chunk: Final = self._next_sse_data()
         try:
-            json_chunk: Final = json.loads(chunk)
+            json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
             return self.chunk_parser(chunk=json_chunk)
         except ValueError as e:
             verbose_logger.debug(
@@ -126,7 +131,7 @@ class ModelResponseIterator:
         if chunk == "[DONE]":
             raise StopAsyncIteration
         try:
-            json_chunk: Final = json.loads(chunk)
+            json_chunk: Final = _JSON_CHUNK.validate_python(json.loads(chunk))
             return self.chunk_parser(chunk=json_chunk)
         except ValueError as e:
             verbose_logger.debug(

litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py

@@ -4,13 +4,14 @@ from typing import Final
 
 from httpx._types import RequestFiles
 
+from litellm.images.utils import image_to_data_url
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.images.main import ImageEditOptionalRequestParams
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import FileTypes
 
-from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig, to_data_url
+from .transformation import DEFAULT_BASE_URL, FalAIImageEditConfig
 
 FLUX_LORA_DEPTH_ENDPOINT: Final[str] = "fal-ai/flux-lora-depth"
 SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("n", "size")
@@ -68,7 +69,7 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig):
         )
         request_body: Final[dict[str, object]] = {  # mutable-ok: base class contract returns a dict
             "prompt": prompt,
-            "image_url": to_data_url(next(iter(images))),
+            "image_url": image_to_data_url(next(iter(images))),
             **provider_params,
         }
         return request_body, ()

litellm/llms/fal_ai/image_edit/transformation.py

@@ -1,15 +1,12 @@
-import base64
-import os
 from collections.abc import Mapping
-from pathlib import Path
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
+from typing import TYPE_CHECKING, Final
 
 import httpx
 from httpx._types import RequestFiles
 from pydantic import ConfigDict, TypeAdapter
 
-from litellm.images.utils import ImageEditRequestUtils
+from litellm.images.utils import image_to_data_url
 from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
 from litellm.llms.fal_ai.image_generation.gpt_image_2_transformation import (
     map_gpt_image_quality,
@@ -39,39 +36,6 @@ PARAM_TRANSLATION: Final[Mapping[str, str]] = MappingProxyType(
 )
 
 
-@runtime_checkable
-class _SeekableBinaryReader(Protocol):
-    def tell(self) -> int: ...
-
-    def seek(self, offset: int) -> int: ...
-
-    def read(self) -> bytes: ...
-
-
-def _read_image_bytes(image: object) -> bytes:
-    if isinstance(image, bytes):
-        return image
-    if isinstance(image, tuple):
-        return _read_image_bytes(image[1])
-    if isinstance(image, os.PathLike):
-        return Path(image).read_bytes()
-    if isinstance(image, _SeekableBinaryReader):
-        position: Final = image.tell()
-        image.seek(0)
-        data: Final = image.read()
-        image.seek(position)
-        return data
-    raise ValueError(f"Unsupported image type for Fal AI image edit: {type(image).__name__}")
-
-
-def to_data_url(image: object) -> str:
-    if isinstance(image, str):
-        return image
-    image_bytes: Final = _read_image_bytes(image)
-    mime_type: Final = ImageEditRequestUtils.get_image_content_type(image_bytes)
-    return f"data:{mime_type};base64,{base64.b64encode(image_bytes).decode('utf-8')}"
-
-
 def _first(value: object) -> object:
     return value[0] if isinstance(value, list) and value else value
 
@@ -146,14 +110,14 @@ class FalAIImageEditConfig(BaseImageEditConfig):
             raise ValueError("Fal AI image edit requires at least one input image")
         mask: Final = _first(image_edit_optional_request_params.get("mask"))
         mask_field: Final[Mapping[str, str]] = (
-            MappingProxyType({"mask_url": to_data_url(mask)}) if mask is not None else MappingProxyType({})
+            MappingProxyType({"mask_url": image_to_data_url(mask)}) if mask is not None else MappingProxyType({})
         )
         provider_params: Final[Mapping[str, object]] = MappingProxyType(
             {key: value for key, value in image_edit_optional_request_params.items() if key != "mask"}
         )
         request_body: Final[dict[str, object]] = {  # mutable-ok: base class contract returns a dict
             "prompt": prompt,
-            "image_urls": tuple(to_data_url(img) for img in images),
+            "image_urls": tuple(image_to_data_url(img) for img in images),
             **mask_field,
             **provider_params,
         }