scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 233

Merge remote-tracking branch 'origin/main' into litellm_bigmodel

Browse source 
# Conflicts:
#	ui/litellm-dashboard/src/components/provider_info_helpers.tsx
This commit is contained in:
mateo-berri 2026-10-28 15:42:50 -07:00
commit 001a5f2320
198 changed files with 14992 additions and 1422 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/workflows/interactions-spec-canary.yml vendored
View file

@ -19,7 +19,7 @@ jobs:
name: interactions-spec-canary
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 15
timeout-minutes: 30
permissions:
contents: read
outputs:
@ -39,8 +39,11 @@ jobs:
with:
version: "0.10.9"
- name: Set up the Rust bridge
uses: ./.github/actions/rust-bridge
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Run the compliance tests against Google's live spec
id: compliance

7
.github/workflows/test-merge-smoke.yml vendored
View file

@ -132,7 +132,7 @@ jobs:
windows-sdk:
name: windows-sdk
runs-on: windows-2025
timeout-minutes: 10
timeout-minutes: 30
env:
LITELLM_LOCAL_MODEL_COST_MAP: "True"
defaults:
@ -154,8 +154,11 @@ jobs:
with:
version: "0.10.9"
- name: Set up the Rust bridge
uses: ./.github/actions/rust-bridge
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --no-install-workspace --group dev --python 3.11
run: .github/scripts/uv_sync_with_retries.sh --frozen --group dev --python 3.11
- name: Run the Windows tests
run: uv run --no-sync python -m pytest --tb=short -v tests/windows_tests

11
.github/workflows/test-unit.yml vendored
View file

@ -137,6 +137,7 @@ jobs:
include:
- shard: core-utils
test-path: |-
tests/unit/accounting
tests/unit/batch_completion
tests/unit/decisions
tests/unit/extraction
@ -1307,6 +1308,8 @@ jobs:
done
openai-sdk-compat:
name: OpenAI ${{ matrix.openai-version }} compatibility
needs: rust-bridge
if: ${{ !cancelled() }}
permissions:
contents: read
pull-requests: read
@ -1343,10 +1346,18 @@ jobs:
with:
version: "0.10.9"
- name: Set up the Rust bridge
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/rust-bridge
with:
artifact: ${{ needs.rust-bridge.outputs.artifact }}
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group dev --group proxy-dev --extra proxy --no-install-workspace
cp rust-bridge-dist/litellm/rust_bridge/_native.* litellm/rust_bridge/
uv pip install "openai==${OPENAI_VERSION:?}" --exclude-newer-package openai=2026-10-07T00:00:00Z
uv run --no-sync python -c 'import os, openai; assert openai.__version__ == os.environ["OPENAI_VERSION"], openai.__version__'

1
README.md
View file

@ -427,6 +427,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | |
| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | |
| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) | | | | ✅ | | | | | | |
| [ZeroGPU (`zerogpu`)](https://docs.litellm.ai/docs/providers/zerogpu) | ✅ | ✅ | ✅ | | | | | | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)

58
ci_cd/spend_logs_guard.py
View file

@ -4,7 +4,9 @@ needs an approving review from the spend-log owner before it can merge.
CODEOWNERS can only match file paths, and every schema change edits the same schema.prisma copies,
so a path rule would put every migration on the owner. This gate reads the diff instead: when no
added line in the watched files mentions the table it passes at once, otherwise it passes only once
the owner's latest review on the pull request is an approval. The workflow re-runs on every review.
the owner's latest review on the pull request is an approval that still covers every spend-log line
in the head (approvals survive pushes, so a later commit may not add spend-log lines the owner never
saw). The workflow re-runs on every review.
"""
from __future__ import annotations
@ -13,7 +15,7 @@ import argparse
import re
import subprocess
import sys
from collections.abc import Sequence
from collections.abc import Callable, Sequence
from pathlib import Path
from typing import Final
@ -25,7 +27,8 @@ WATCHED: Final = (
"litellm-proxy-extras/litellm_proxy_extras/schema.prisma",
)
MIGRATIONS_PREFIX: Final = "litellm-proxy-extras/litellm_proxy_extras/migrations/"
TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs", re.IGNORECASE)
TABLE: Final = re.compile(r"(?<![A-Za-z0-9])LiteLLM_SpendLogs(?![A-Za-z0-9])", re.IGNORECASE)
DECIDING_STATES: Final = frozenset({"APPROVED", "CHANGES_REQUESTED", "DISMISSED"})
class Reviewer(BaseModel):
@ -37,6 +40,7 @@ class Review(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
state: str
user: Reviewer
commit_id: str | None = None
_REVIEWS: Final = TypeAdapter(list[Review])
@ -70,11 +74,33 @@ def _path_of(section: str) -> str:
def approved_by(reviews: Sequence[Review], login: str) -> bool:
"""True when `login`'s most recent review on the pull request is an approval."""
states: Final = tuple(review.state for review in reviews if review.user.login == login)
"""True when `login`'s most recent review that changes GitHub's approval state is an approval.
COMMENTED and PENDING reviews leave an earlier approval in force on GitHub, so they are skipped here too.
"""
own: Final = tuple(review.state for review in reviews if review.user.login == login)
states: Final = tuple(state for state in own if state in DECIDING_STATES)
return bool(states) and states[-1] == "APPROVED"
def latest_approval(reviews: Sequence[Review], login: str) -> Review | None:
"""`login`'s most recent APPROVED review on the pull request, the one GitHub keeps in force after pushes."""
own: Final = tuple(review for review in reviews if review.user.login == login and review.state == "APPROVED")
return own[-1] if own else None
def approval_covers(
approval: Review, head: str, hits: Sequence[str], hits_at: Callable[[str], Sequence[str] | None]
) -> bool:
"""True when the approval is on the head, or the head adds no spend-log line the approved commit did not."""
if approval.commit_id is None:
return False
if approval.commit_id == head:
return True
reviewed: Final = hits_at(approval.commit_id)
return reviewed is not None and set(hits) <= set(reviewed)
def added_diff(base: str, head: str) -> str:
result: Final = subprocess.run(
("git", "diff", "--unified=0", "--no-color", f"{base}..{head}", "--", *WATCHED, MIGRATIONS_PREFIX),
@ -85,6 +111,15 @@ def added_diff(base: str, head: str) -> str:
return result.stdout
def fetched(commit: str) -> bool:
result: Final = subprocess.run(("git", "fetch", "--no-tags", "--depth=1", "origin", commit), check=False)
return result.returncode == 0
def reviewed_hits(base: str, commit: str) -> tuple[str, ...] | None:
return spend_log_hits(added_diff(base, commit)) if fetched(commit) else None
class Arguments(BaseModel):
model_config = ConfigDict(frozen=True)
base: str
@ -104,14 +139,21 @@ def main(argv: Sequence[str]) -> int:
if not hits:
sys.stdout.write("No added line in schema.prisma or a migration names LiteLLM_SpendLogs; nothing to review.\n")
return 0
if approved_by(_REVIEWS.validate_json(args.reviews.read_bytes()), args.approver):
reviews: Final = _REVIEWS.validate_json(args.reviews.read_bytes())
approval: Final = latest_approval(reviews, args.approver) if approved_by(reviews, args.approver) else None
if approval is not None and approval_covers(
approval, args.head, hits, lambda commit: reviewed_hits(args.base, commit)
):
sys.stdout.write(f"LiteLLM_SpendLogs change approved by {args.approver}: {', '.join(hits)}\n")
return 0
listed: Final = "".join(f" {hit}\n" for hit in hits)
sys.stdout.write(
reason: Final = (
f"This pull request changes LiteLLM_SpendLogs and needs an approving review from @{args.approver}:\n"
f"{listed}The check re-runs when a review is submitted.\n"
if approval is None
else f"LiteLLM_SpendLogs lines changed after @{args.approver} approved {approval.commit_id}, "
f"a new approving review is needed:\n"
)
sys.stdout.write(f"{reason}{listed}The check re-runs when a review is submitted.\n")
return 1

8
enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py
View file

@ -32,6 +32,7 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.llms.base_llm.guardrail_translation.utils import effective_skip_tool_message_for_guardrail
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_tool_json_fields, map_tool_json_text, walk_user_text
from litellm.types.guardrails import GuardrailEventHooks
@ -753,7 +754,12 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
_masked_entity_count.set({})
# Covers multimodal list content + Responses-API input.
inspected = walk_user_text(data, self.redact_text, self.redact_tool_payload) + self._redact_prompt(data)
inspected = walk_user_text(
data,
self.redact_text,
self.redact_tool_payload,
skip_tool_message=effective_skip_tool_message_for_guardrail(self),
) + self._redact_prompt(data)
if inspected == 0:
# Image-only, empty-text, and unsupported payloads inspected

171
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -5,9 +5,9 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
import asyncio
import json
import re
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone
from datetime import datetime, timedelta
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast
@ -28,6 +28,7 @@ from litellm.constants import (
from litellm.repositories.table_repositories import ManagedObjectRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.utils import get_utc_datetime
if TYPE_CHECKING:
from prisma import models as prisma_models
@ -42,11 +43,14 @@ if TYPE_CHECKING:
def _error_names_resource(error: Exception, resource_id: str) -> bool:
return resource_id in re.findall(r"""[^\s"'`]+""", str(error))
tokens: Final[list[str]] = re.findall(r"""[^\s"'`<>(]+""", str(error))
return any(token.rstrip(".,:;)") == resource_id for token in tokens)
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48)
GIVE_UP_POLLING_AFTER: Final = timedelta(days=2 * MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
"completed",
@ -91,6 +95,10 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li
return TeamRepository(prisma_client).table
def _unique_jobs(jobs: Sequence[ManagedObjectRow]) -> tuple[ManagedObjectRow, ...]:
return tuple({job.id: job for job in jobs}.values())
class CheckBatchCost:
def __init__(
self,
@ -98,6 +106,7 @@ class CheckBatchCost:
prisma_client: "PrismaClient",
llm_router: "Router",
track_unmanaged_batch_cost: bool = False,
clock: Callable[[], datetime] = get_utc_datetime,
):
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
@ -106,13 +115,14 @@ class CheckBatchCost:
self.prisma_client: PrismaClient = prisma_client
self.llm_router: Router = llm_router
self._track_unmanaged_batch_cost = track_unmanaged_batch_cost
self._clock: Final = clock
self._creator_attribution: Final = ManagedObjectCreatorAttribution(prisma_client, "CheckBatchCost")
# Cached after the first poll cycle. Once we know the column is absent we skip
# the guaranteed-failing primary query on every subsequent cycle.
self._has_batch_processed_column: bool = True
self.batch_processed_support_confirmed: bool = False
self._poll_cursor: tuple[datetime, str] | None = None
self._poll_before: datetime = datetime.now(timezone.utc)
self._poll_before: datetime = clock()
self._arrival_cursor: tuple[datetime, str] | None = None
self._arrival_before: datetime = self._poll_before
self._poll_arrivals: bool = False
@ -147,27 +157,6 @@ class CheckBatchCost:
return
self.batch_processed_support_confirmed = True
async def _cleanup_stale_managed_objects(self) -> None:
"""
Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
in non-terminal states as 'stale_expired'. These will never complete and
should not be polled.
"""
cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
result: Final = await _managed_object_table(self.prisma_client).update_many(
where={
"file_purpose": "batch",
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
"created_at": {"lt": cutoff},
},
data={"status": "stale_expired"},
)
if result > 0:
verbose_proxy_logger.warning(
f"CheckBatchCost: marked {result} stale managed objects "
f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
)
async def _fallback_find_jobs(self) -> "Sequence[ManagedObjectRow]":
"""Query batch jobs without the batch_processed filter (for older schemas)."""
return await self._find_jobs(
@ -187,20 +176,43 @@ class CheckBatchCost:
)
async def _find_jobs(self, where: Mapping[str, object]) -> Sequence[ManagedObjectRow]:
arrivals: Final = self._poll_arrivals
page: Final = await self._find_job_page(where, arrivals=arrivals)
self._poll_arrivals = not arrivals if page else arrivals
if page:
return page
return await self._find_job_page(where, arrivals=not arrivals)
lead_arrivals: Final = self._poll_arrivals
self._poll_arrivals = not lead_arrivals
lead_share: Final = MAX_OBJECTS_PER_POLL_CYCLE - MAX_OBJECTS_PER_POLL_CYCLE // 2
lead: Final = await self._find_job_page(where, arrivals=lead_arrivals, take=lead_share)
trail: Final = await self._find_job_page(
where, arrivals=not lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead)
)
spill: Final = (
await self._find_job_page(
where, arrivals=lead_arrivals, take=MAX_OBJECTS_PER_POLL_CYCLE - len(lead) - len(trail)
)
if len(lead) == lead_share
else ()
)
return _unique_jobs((*lead, *trail, *spill))
async def _find_job_page(self, where: Mapping[str, object], *, arrivals: bool) -> Sequence[ManagedObjectRow]:
async def _find_job_page(
self, where: Mapping[str, object], *, arrivals: bool, take: int
) -> Sequence[ManagedObjectRow]:
if take == 0:
return ()
resumed_mid_lane: Final = (self._arrival_cursor if arrivals else self._poll_cursor) is not None
page: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take)
if len(page) == take or not resumed_mid_lane:
return page
wrapped: Final = await self._find_job_page_from_cursor(where, arrivals=arrivals, take=take - len(page))
return _unique_jobs((*page, *wrapped))
async def _find_job_page_from_cursor(
self, where: Mapping[str, object], *, arrivals: bool, take: int
) -> Sequence[ManagedObjectRow]:
cursor: Final = self._arrival_cursor if arrivals else self._poll_cursor
if cursor is None:
if arrivals:
self._arrival_before = datetime.now(timezone.utc)
self._arrival_before = self._clock()
else:
self._poll_before = datetime.now(timezone.utc) - RECENT_BATCH_POLL_WINDOW
self._poll_before = self._clock() - RECENT_BATCH_POLL_WINDOW
before: Final = self._arrival_before if arrivals else self._poll_before
page: Final = await _managed_object_table(self.prisma_client).find_many(
where={
@ -220,10 +232,10 @@ class CheckBatchCost:
else {}
),
},
take=MAX_OBJECTS_PER_POLL_CYCLE,
take=take,
order=[{"created_at": "asc"}, {"id": "asc"}],
)
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == MAX_OBJECTS_PER_POLL_CYCLE else None
next_cursor: Final = (page[-1].created_at, page[-1].id) if len(page) == take else None
if arrivals:
self._arrival_cursor = next_cursor
else:
@ -231,11 +243,47 @@ class CheckBatchCost:
return page
def _unprocessed_snapshot(self, job: ManagedObjectRow) -> Mapping[str, object]:
return {
"id": job.id,
"updated_at": job.updated_at,
**({"batch_processed": False} if self._has_batch_processed_column else {}),
}
if self._has_batch_processed_column:
return {"id": job.id, "batch_processed": False}
return {"id": job.id, "updated_at": job.updated_at}
def _is_past_give_up_age(self, job: ManagedObjectRow) -> bool:
return job.created_at < self._clock() - GIVE_UP_POLLING_AFTER
async def _expire_if_old_and_unfinished(self, job: ManagedObjectRow) -> None:
if job.status in TERMINAL_MANAGED_OBJECT_STATUSES or not self._is_past_give_up_age(job):
return
await self._expire_unfinished(job)
@staticmethod
def _is_completed_with_output_id_pending(response: "LiteLLMBatch") -> bool:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
return (
response.status in ("completed", "complete")
and response.output_file_id is None
and not _completed_batch_safe_to_retire(response)
)
async def _expire_unfinished(self, job: ManagedObjectRow) -> None:
try:
expired: Final = await _managed_object_table(self.prisma_client).update_many(
where={
**self._unprocessed_snapshot(job),
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
},
data={"status": "stale_expired"},
)
except Exception as db_err:
verbose_proxy_logger.error(f"CheckBatchCost: failed to mark stale job {job.id} as stale_expired: {db_err}")
return
if expired > 0:
verbose_proxy_logger.warning(
f"CheckBatchCost: job {job.id} is older than {GIVE_UP_POLLING_AFTER.days} days "
"and still not finished, so it is marked stale_expired and no longer polled"
)
async def _retire_job(self, job: "ManagedObjectRow", reason: str) -> None:
"""
@ -356,15 +404,21 @@ class CheckBatchCost:
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
"""A 404 naming the output file means there is nothing to fetch on this or any
later poll: providers like Vertex AI advertise an output path for every batch,
including terminal ones that never wrote it. Any other failure may be
transient, so it keeps retrying until the staleness sweep bounds it."""
including terminal ones that never wrote it. GCS names a gs:// object without its scheme,
and S3 names an s3:// object by its key alone. Any other failure keeps retrying."""
import openai
from litellm.exceptions import NotFoundError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
if not output_file_id:
return False
return isinstance(error, (NotFoundError, openai.NotFoundError)) and _error_names_resource(error, output_file_id)
is_not_found: Final = isinstance(error, (NotFoundError, openai.NotFoundError)) or (
isinstance(error, BaseLLMException) and error.status_code == 404
)
object_path: Final = output_file_id.split("://", 1)[-1]
names: Final = (output_file_id, object_path, object_path.split("/", 1)[-1])
return is_not_found and any(_error_names_resource(error, name) for name in names)
async def _finalize_unbilled_terminal_job(
self, job: "ManagedObjectRow", response: "LiteLLMBatch"
@ -1094,13 +1148,6 @@ class CheckBatchCost:
processed_models: List[Tuple[Optional[str], Optional[str]]] = []
try:
await self._cleanup_stale_managed_objects()
except Exception as cleanup_err:
verbose_proxy_logger.warning(
f"CheckBatchCost: stale cleanup failed (poll will continue): {cleanup_err}"
)
# Look for all batches that have not yet been processed by CheckBatchCost.
# self._has_batch_processed_column is cached after the first probe so that
# older schemas don't pay a guaranteed-failing primary query + warning on
@ -1142,6 +1189,8 @@ class CheckBatchCost:
if routing is None:
if self._has_unified_id_without_model(job):
await self._retire_job(job, "unified object id has no model id")
else:
await self._expire_if_old_and_unfinished(job)
continue
model_id, batch_id = routing
@ -1166,6 +1215,14 @@ class CheckBatchCost:
prom_logger.record_check_batch_cost_error("provider_retrieval_error")
if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id):
await self._retire_job(job, f"batch {batch_id} no longer exists at the provider")
else:
await self._expire_if_old_and_unfinished(job)
continue
if self._is_completed_with_output_id_pending(response):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
continue
## RETRIEVE THE BATCH JOB OUTPUT FILE
@ -1217,17 +1274,9 @@ class CheckBatchCost:
self._store_batch_line_items_in_background(job, response, model_id, batch_id, billed=True)
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
continue
await self._retire_unbilled_terminal_job(job, response, model_id, batch_id)
else:
await self._expire_if_old_and_unfinished(job)
# Record polling run metrics (always, even if nothing was processed)
if prom_logger:

161
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -6,6 +6,7 @@ import base64
import json
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from datetime import datetime, timedelta
from itertools import chain
from types import MappingProxyType
from typing import (
@ -27,7 +28,7 @@ import httpx
from fastapi import HTTPException
from openai import APIConnectionError, AsyncOpenAI
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, Unpack
import litellm
@ -39,6 +40,7 @@ from litellm.constants import (
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS,
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY,
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_FILE_LIST_LIMIT,
PENDING_RESPONSE_STATUSES,
)
@ -51,7 +53,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_metadata,
)
from litellm.llms.base_llm.files.storage_backend_factory import get_storage_backend
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
from litellm.llms.base_llm.managed_resources.isolation import (
build_list_page,
build_owner_filter,
@ -113,6 +115,7 @@ from litellm.types.utils import (
LLMResponseTypes,
SpecialEnums,
)
from litellm.utils import get_utc_datetime
class _ManagedFileRetrieve(Protocol):
@ -306,6 +309,7 @@ def _is_xai_batch_output(
class _ManagedFileRow(Protocol):
unified_file_id: str
file_object: OpenAIFileObject
model_mappings: Mapping[str, str]
flat_model_file_ids: Sequence[str]
storage_backend: Optional[str]
storage_url: Optional[str]
@ -359,6 +363,8 @@ class _ManagedResourceDatabase(Protocol):
@property
def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
async def query_raw(self, query: str, *args: object) -> object: ...
@property
def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
@ -382,10 +388,83 @@ class _RouterFileCallKwargs(TypedDict, total=False):
custom_llm_provider: ReadOnly[str | None]
def _managed_database(prisma_client: _ManagedResourcePrismaClient) -> _ManagedResourceDatabase:
return prisma_client.db
def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _ManagedFileTableActions:
return prisma_client.db.litellm_managedfiletable
class _ListedFileRow(Protocol):
@property
def unified_file_id(self) -> str: ...
@property
def file_object(self) -> object: ...
@property
def model_mappings(self) -> Mapping[str, object]: ...
@property
def storage_backend(self) -> str | None: ...
class _ScopedFileRow(BaseModel):
model_config = ConfigDict(frozen=True)
unified_file_id: str
file_object: JsonValue = None
model_mappings: Mapping[str, JsonValue]
storage_backend: str | None = None
_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...])
def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]:
match owner_filter:
case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}:
return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id)
case {"created_by": str() as user_id}:
return '"created_by" = $1', (user_id,)
case {"team_id": str() as team_id}:
return '"team_id" = $1', (team_id,)
case _:
return ("FALSE", ()) if owner_filter else ("TRUE", ())
async def _scoped_file_rows(
database: _ManagedResourceDatabase,
owner_filter: Mapping[str, object],
model_mapping_keys: frozenset[str],
cursor_id: str | None,
direction: Literal["asc", "desc"],
take: int,
) -> tuple[_ScopedFileRow, ...]:
owner_condition, owner_params = _owner_condition(owner_filter)
keys_param: Final = len(owner_params) + 1
comparison: Final = "<" if direction == "desc" else ">"
cursor_condition: Final = (
f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" '
f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) '
if cursor_id
else ""
)
order: Final = direction.upper()
rows: Final = await database.query_raw(
'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" '
f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) '
f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} '
f"LIMIT ${keys_param + (2 if cursor_id else 1)}",
*owner_params,
sorted(model_mapping_keys),
*((cursor_id,) if cursor_id else ()),
take,
)
return _SCOPED_FILE_ROWS.validate_python(rows)
def _iter_provider_file_id_pairs(
rows: Sequence[_ManagedFileRow],
requested_provider_file_ids: frozenset[str],
@ -507,10 +586,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
prisma_client: PrismaClient,
*,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
clock: Callable[[], datetime] = get_utc_datetime,
):
self.internal_usage_cache = internal_usage_cache
self.prisma_client = prisma_client
self._sleep = sleep
self._clock = clock
@staticmethod
def _get_prometheus_logger():
@ -2260,14 +2341,23 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
model_scope: Optional[FileListModelScope] = None,
before: Optional[str] = None,
**data: Dict,
) -> FileListPage:
"""List the managed files the caller owns, newest first.
Pagination is keyset based on ``unified_file_id`` so a key that owns
every file on the proxy still reads one bounded page at a time.
``purpose`` is applied after parsing, because the managed file table
keeps it inside the ``file_object`` blob instead of a column, and rows
``after`` pages toward older files and ``before`` toward newer ones,
each page still ordered newest first.
``model_scope`` is applied in the query, so only rows whose
``model_mappings`` name one of its deployment ids or model names, and
storage-backed rows, leave the database; a storage-backed row then
stays only when its model name is one the router sends to the listed
provider. ``purpose`` is applied after parsing, because the managed
file table keeps it inside the ``file_object`` blob instead of a
column. Rows
whose blob will not parse drop out there too, so a chunk of rows can
yield fewer matches than the page holds. Successive chunks are read
until the page is full or the caller's rows run out, which keeps
@ -2276,9 +2366,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
scan has to continue past it, the chunk widens to
``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
that many rows instead of one per page. That bound is per query, not
per request: the work is still linear in the rows the caller owns, and
a filter matching nothing reads every one of them, with no index
covering either the owner filter or the sort.
per request: a ``purpose`` filter matching nothing still reads every
row that passes the query.
"""
validate_file_list_limit(limit)
validate_file_list_purpose(purpose)
@ -2287,36 +2376,60 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if owner_filter is None:
return FileListPage.model_validate(build_list_page([]))
if after:
cursor_row = await _managed_file_table(self.prisma_client).find_first(
where={**owner_filter, "unified_file_id": after}
if after and before:
raise ProxyException(
message="Pass either 'after' (or 'after_id') or 'before_id' when listing files, not both.",
type="invalid_request_error",
param="before_id",
code=400,
openai_code="invalid_value",
)
cursor_param: Final = "before_id" if before else "after"
cursor: Final = before or after
database: Final = _managed_database(self.prisma_client)
if cursor:
cursor_row = await database.litellm_managedfiletable.find_first(
where={**owner_filter, "unified_file_id": cursor}
)
if cursor_row is None:
raise ProxyException(
message=f"Invalid 'after' cursor: no file found with id '{after}'.",
message=f"Invalid '{cursor_param}' cursor: no file found with id '{cursor}'.",
type="invalid_request_error",
param="after",
param=cursor_param,
code=400,
openai_code="invalid_value",
)
if model_scope is not None and not model_scope.model_mapping_keys:
return FileListPage.model_validate(build_list_page([]))
page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
matches: Final[List[OpenAIFileObject]] = []
cursor_id = after
cursor_id = cursor
chunk_size = page_size + 1
direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc"
while len(matches) <= page_size:
cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
chunk = await _managed_file_table(self.prisma_client).find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
**cursor_args,
chunk: Sequence[_ListedFileRow] = (
await _scoped_file_rows(
database, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size
)
if model_scope is not None
else await database.litellm_managedfiletable.find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": direction}, {"unified_file_id": direction}],
**cursor_args,
)
)
matches.extend(
_public_file_object(parsed_file_object, row.unified_file_id)
for row in chunk
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
if (
model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
)
and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)
)
if len(chunk) < chunk_size:
@ -2324,7 +2437,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
cursor_id = chunk[-1].unified_file_id
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size))
page: Final = matches[:page_size]
return FileListPage.model_validate(
build_list_page(page[::-1] if before else page, has_more=len(matches) > page_size)
)
def _is_batch_polling_enabled(self) -> bool:
"""
@ -2387,12 +2503,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
where={
"file_purpose": "batch",
"batch_processed": False,
"status": {"not_in": ["failed", "expired", "cancelled"]},
"status": {"not_in": ["failed", "expired", "cancelled", "stale_expired"]},
},
take=MAX_MATCHES_TO_RETURN,
order={"created_at": "desc"},
)
input_file_cutoff: Final = self._clock() - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
referencing_batches: Final[list[dict[str, object]]] = []
for batch in batches:
try:
@ -2405,7 +2522,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Extract file IDs from batch
# Batches typically reference the unified file ID in input_file_id
# Output and error files are generated by the provider
input_file_id = batch_data.get("input_file_id")
input_file_id = batch_data.get("input_file_id") if batch.created_at >= input_file_cutoff else None
output_file_id = batch_data.get("output_file_id")
error_file_id = batch_data.get("error_file_id")

2
gateway/Dockerfile
View file

@ -153,7 +153,7 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
find /app/.venv -type d -path "*/tornado/test" -delete && \
chmod -R a+rX /opt/prisma && \
python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
python -c "import litellm, litellm.rust_bridge._native; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__"
USER nonroot

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261021000000_managed_file_created_by_created_at_index/migration.sql Normal file
View file

@ -0,0 +1,2 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_ManagedFileTable_created_by_created_at_idx"
ON "LiteLLM_ManagedFileTable" ("created_by", "created_at" DESC);

47
litellm-proxy-extras/litellm_proxy_extras/migrations/20261027000000_add_accounting_ledger_tables/migration.sql Normal file
View file

@ -0,0 +1,47 @@
SET LOCAL lock_timeout = '10s';
CREATE TYPE "AccountingEventKind" AS ENUM ('inbound_usage', 'outbound_usage', 'cost');
CREATE TYPE "AccountingDeliveryState" AS ENUM ('pending', 'applied', 'parked');
CREATE TYPE "AccountingDeliveryTarget" AS ENUM ('LiteLLM_VerificationToken', 'LiteLLM_UserTable', 'LiteLLM_TeamTable', 'LiteLLM_TeamMembership', 'LiteLLM_OrganizationTable', 'LiteLLM_OrganizationMembership', 'LiteLLM_EndUserTable', 'LiteLLM_AgentsTable', 'LiteLLM_TagTable', 'LiteLLM_ProjectTable', 'LiteLLM_ModelAccessGroupBudgetTable', 'LiteLLM_BudgetTable', 'LiteLLM_BudgetWindowSpend', 'LiteLLM_DailyUserSpend', 'LiteLLM_DailyTeamSpend', 'LiteLLM_DailyOrganizationSpend', 'LiteLLM_DailyEndUserSpend', 'LiteLLM_DailyAgentSpend', 'LiteLLM_DailyTagSpend', 'LiteLLM_DailyModelUsage', 'LiteLLM_DailyToolSpend', 'LiteLLM_AutoRouterDailySpend');
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingEvent" (
"event_id" TEXT NOT NULL,
"kind" "AccountingEventKind" NOT NULL,
"source_event_id" TEXT,
"schema_version" INTEGER NOT NULL,
"payload_hash" TEXT NOT NULL,
"occurred_at" TIMESTAMP(3) NOT NULL,
"recorded_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"payload" JSONB NOT NULL,
CONSTRAINT "LiteLLM_AccountingEvent_schema_version_check" CHECK ("schema_version" > 0),
CONSTRAINT "LiteLLM_AccountingEvent_pkey" PRIMARY KEY ("event_id")
);
-- CreateTable
CREATE TABLE IF NOT EXISTS "LiteLLM_AccountingDelivery" (
"event_id" TEXT NOT NULL,
"target" "AccountingDeliveryTarget" NOT NULL,
"version" INTEGER NOT NULL,
"state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
"attempts" INTEGER NOT NULL DEFAULT 0,
"next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"applied_at" TIMESTAMP(3),
"last_error" TEXT,
CONSTRAINT "LiteLLM_AccountingDelivery_version_check" CHECK ("version" > 0),
CONSTRAINT "LiteLLM_AccountingDelivery_attempts_check" CHECK ("attempts" >= 0),
CONSTRAINT "LiteLLM_AccountingDelivery_pkey" PRIMARY KEY ("event_id","target","version")
);
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_recorded_at_idx" ON "LiteLLM_AccountingEvent"("recorded_at");
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_source_event_id_idx" ON "LiteLLM_AccountingEvent"("kind", "source_event_id");
-- CreateIndex
CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingDelivery_target_state_next_attempt_at_idx" ON "LiteLLM_AccountingDelivery"("target", "state", "next_attempt_at");

71
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
@@index([unified_file_id])
@@index([flat_model_file_ids], type: Gin)
@@index([team_id, created_at(sort: Desc)])
@@index([created_by, created_at(sort: Desc)])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
@@index([window_end])
@@index([created_at])
}
enum AccountingEventKind {
inbound_usage
outbound_usage
cost
}
enum AccountingDeliveryState {
pending
applied
parked
}
enum AccountingDeliveryTarget {
LiteLLM_VerificationToken
LiteLLM_UserTable
LiteLLM_TeamTable
LiteLLM_TeamMembership
LiteLLM_OrganizationTable
LiteLLM_OrganizationMembership
LiteLLM_EndUserTable
LiteLLM_AgentsTable
LiteLLM_TagTable
LiteLLM_ProjectTable
LiteLLM_ModelAccessGroupBudgetTable
LiteLLM_BudgetTable
LiteLLM_BudgetWindowSpend
LiteLLM_DailyUserSpend
LiteLLM_DailyTeamSpend
LiteLLM_DailyOrganizationSpend
LiteLLM_DailyEndUserSpend
LiteLLM_DailyAgentSpend
LiteLLM_DailyTagSpend
LiteLLM_DailyModelUsage
LiteLLM_DailyToolSpend
LiteLLM_AutoRouterDailySpend
}
// Durable accounting ledger. One row per usage or cost event; payload holds the
// private event envelope versioned by schema_version. Usage rows retain request-scoped
// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
model LiteLLM_AccountingEvent {
event_id String @id
kind AccountingEventKind
source_event_id String?
schema_version Int
payload_hash String
occurred_at DateTime
recorded_at DateTime @default(now())
payload Json
@@index([recorded_at])
@@index([kind, source_event_id])
}
// One row per (event, target table, version): pending work for the projector and,
// once applied in the same transaction as the target write, the receipt that stops a retry.
model LiteLLM_AccountingDelivery {
event_id String
target AccountingDeliveryTarget
version Int
state AccountingDeliveryState @default(pending)
attempts Int @default(0)
next_attempt_at DateTime @default(now())
applied_at DateTime?
last_error String?
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}

2
litellm-rust/crates/core-utils/src/core_helpers.rs
View file

@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
("max_tokens", "length"),
("refusal", "content_filter"),
("compaction", "length"),
("model_context_window_exceeded", "length"),
("guardrail_intervened", "content_filter"),
("content_filtered", "content_filter"),
("content_filter", "content_filter"),
@ -74,6 +75,7 @@ mod tests {
#[case::end_turn("end_turn", "stop")]
#[case::stop_sequence("stop_sequence", "stop")]
#[case::max_tokens("max_tokens", "length")]
#[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
#[case::refusal("refusal", "content_filter")]
#[case::guardrail_intervened("guardrail_intervened", "content_filter")]
#[case::content_filtered("content_filtered", "content_filter")]

1
litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs
View file

@ -113,6 +113,7 @@ const OPENAI_COMPATIBLE_PROVIDERS: &[&str] = &[
"meta",
"cognition",
"scx-ai",
"zerogpu",
];
impl ExceptionFamily {

2
litellm-rust/crates/python-bridge/src/logger/tests.rs
View file

@ -187,6 +187,7 @@ fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication
locals
.set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
.unwrap();
locals.set_item("native", crate::native_module(py)).unwrap();
let importable = py
.eval(
c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
@ -205,6 +206,7 @@ import asyncio
import logging
import sys
sys.path.insert(0, repo_root)
sys.modules['litellm.rust_bridge._native'] = native
import litellm
from litellm._logging import verbose_logger, session_id_var, trace_id_var

9
litellm/__init__.py
View file

@ -682,6 +682,7 @@ moonshot_models: Set = set()
publicai_models: Set = set()
darkbloom_models: Set = set()
openinfer_models: Final[Set[str]] = set()
zerogpu_models: Final[Set[str]] = set()
v0_models: Set = set()
morph_models: Set = set()
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
@ -972,6 +973,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
darkbloom_models.add(key)
elif value.get("litellm_provider") == "openinfer":
openinfer_models.add(key)
elif value.get("litellm_provider") == "zerogpu":
zerogpu_models.add(key)
elif value.get("litellm_provider") == "v0":
v0_models.add(key)
elif value.get("litellm_provider") == "morph":
@ -1146,6 +1149,7 @@ model_list = list(
| publicai_models
| darkbloom_models
| openinfer_models
| zerogpu_models
| v0_models
| morph_models
| opencode_go_models
@ -1266,6 +1270,7 @@ def _build_models_by_provider() -> dict:
"publicai": publicai_models,
"darkbloom": darkbloom_models,
"openinfer": openinfer_models,
"zerogpu": zerogpu_models,
"v0": v0_models,
"morph": morph_models,
"opencode_go": opencode_go_models,
@ -1454,6 +1459,7 @@ from .exceptions import (
Timeout,
APIConnectionError,
UnsupportedParamsError,
UnsupportedEndpointError as UnsupportedEndpointError,
APIResponseValidationError,
UnprocessableEntityError,
InternalServerError,
@ -1743,6 +1749,9 @@ if TYPE_CHECKING:
from .llms.lm_studio.decisions.transformation import (
LMStudioDecisionsConfig as LMStudioDecisionsConfig,
)
from .llms.ollama.decisions.transformation import (
OllamaDecisionsConfig as OllamaDecisionsConfig,
)
from .llms.laya.decisions.transformation import (
LayaDecisionsConfig as LayaDecisionsConfig,
)

2
litellm/_lazy_imports_registry.py
View file

@ -171,6 +171,7 @@ LLM_CONFIG_NAMES: Final = (
"DatabricksDecisionsConfig",
"HostedVLLMDecisionsConfig",
"LMStudioDecisionsConfig",
"OllamaDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"OpenAIDecisionsConfig",
@ -749,6 +750,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"HostedVLLMDecisionsConfig",
),
"LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"),
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),

62
litellm/accounting/__init__.py Normal file
View file

@ -0,0 +1,62 @@
"""Public accounting contracts. Importing these values does not enable accounting."""
from litellm.accounting.measurements import measurements_for
from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter
from litellm.accounting.types import (
AdmissionDecision,
AggregateQuery,
AggregateRow,
CostEvent,
DeliveryState,
DeliveryTarget,
Dimension,
DimensionFilter,
DimensionName,
EventKind,
Granularity,
InboundUsageEvent,
Measurement,
Metric,
MetricName,
Multiplier,
NamedCost,
OutboundUsageEvent,
Outcome,
PricedQuantity,
PricingSnapshot,
Rate,
Unpriced,
validate_aggregate_rows,
)
__all__ = [
"AdmissionDecision",
"AggregateQuery",
"AggregateRow",
"BudgetEnforcer",
"CostAggregator",
"CostCompiler",
"CostEvent",
"DeliveryState",
"DeliveryTarget",
"Dimension",
"DimensionFilter",
"DimensionName",
"EventKind",
"Granularity",
"InboundUsageEvent",
"Measurement",
"Metric",
"MetricName",
"Multiplier",
"NamedCost",
"OutboundUsageEvent",
"Outcome",
"PricedQuantity",
"PricingSnapshot",
"Rate",
"Unpriced",
"UsageMeter",
"measurements_for",
"validate_aggregate_rows",
]

13
litellm/accounting/measurements.py Normal file
View file

@ -0,0 +1,13 @@
from typing import Final
from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent
def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]:
"""Return the complete contribution set for one event, ready for one apply call."""
metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics
occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at
return tuple(
Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric)
for metric in metrics or ()
)

85
litellm/accounting/protocols.py Normal file
View file

@ -0,0 +1,85 @@
"""Contracts only. Implementations own persistence, pricing, projections and live budgets.
The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
"""
from typing import Protocol
from litellm.accounting.types import (
AdmissionDecision,
AggregateQuery,
AggregateRow,
CostEvent,
Dimension,
InboundUsageEvent,
Measurement,
Metric,
OutboundUsageEvent,
Unpriced,
)
class UsageMeter(Protocol):
async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
"""Acknowledge commit, no-op identical IDs and reject conflicting content.
Persist the event and its request-scoped pricing context before acknowledging.
HTTP acknowledgement is a separate policy. Unknown usage remains None.
"""
...
class CostCompiler(Protocol):
async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
"""Reuse the committed charge or finalize one charge and its delivery intent.
Resolve captured request-scoped context by event identity and cost-map rates
at compilation. Persist the resolved inputs in the required snapshot. Replay
reuses the committed result and never consumes free allowance again.
Missing pricing is Unpriced, never an inferred zero.
"""
...
class CostAggregator(Protocol):
async def apply(self, measurements: tuple[Measurement, ...]) -> None:
"""Apply one source event's complete contribution set per target/version.
Each target commits all contributions and its receipt atomically. A receipt
for the first metric must not suppress the rest. Usage need not wait for cost.
"""
...
async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
"""Return disjoint [start, end) rows per group using existing daily projections.
Filters combine with AND, values within a filter with IN. DAY emits daily
buckets; None emits whole-interval totals. Reject unsupported metric/scope
combinations. Different groups may overlap, including non-additive tags.
"""
...
class BudgetEnforcer(Protocol):
async def reserve(
self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...]
) -> AdmissionDecision:
"""Apply existing admission policies and estimates; duplicate IDs add no hold."""
...
async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None:
"""Settle the committed charge with atomic deduplication and hold mutation.
Preserve scope/window attribution and retry unfinished settlement after
restart. SQL projection receipts do not provide live-counter atomicity.
"""
...
async def release(self, reservation_id: str) -> None:
"""Idempotently release unused exposure at the request boundary, preserving charges."""
...

428
litellm/accounting/types.py Normal file
View file

@ -0,0 +1,428 @@
"""Accounting values use UTC instants, decimal strings on the wire and USD charges."""
from collections.abc import Mapping
from datetime import datetime, timezone
from decimal import Decimal
from enum import Enum
from typing import Annotated, Final, Literal, TypeAlias
from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator
from typing_extensions import Self
from litellm.types.utils import CallTypes
_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True)
Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")]
Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)]
def _utc(value: datetime) -> datetime:
return value.astimezone(timezone.utc)
Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)]
class MetricName(str, Enum):
"""Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens."""
REQUESTS = "requests"
SUCCESSFUL_REQUESTS = "successful_requests"
FAILED_REQUESTS = "failed_requests"
PROMPT_TOKENS = "prompt_tokens"
COMPLETION_TOKENS = "completion_tokens"
REASONING_TOKENS = "reasoning_tokens"
TOTAL_TOKENS = "total_tokens"
CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens"
CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens"
CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr"
SPEND = "spend"
class DimensionName(str, Enum):
KEY = "key"
USER = "user"
TEAM = "team"
ORGANIZATION = "organization"
END_USER = "end_user"
MODEL = "model"
MODEL_GROUP = "model_group"
DEPLOYMENT = "deployment"
PROVIDER = "provider"
TOOL = "tool"
AGENT = "agent"
PROJECT = "project"
MODEL_ACCESS_GROUP = "model_access_group"
API_BASE = "api_base"
SERVICE_TIER = "service_tier"
TAG = "tag"
class Outcome(str, Enum):
SUCCEEDED = "succeeded"
FAILED = "failed"
CANCELLED = "cancelled"
class Granularity(str, Enum):
DAY = "day"
class EventKind(str, Enum):
INBOUND_USAGE = "inbound_usage"
OUTBOUND_USAGE = "outbound_usage"
COST = "cost"
class DeliveryState(str, Enum):
PENDING = "pending"
APPLIED = "applied"
PARKED = "parked"
class DeliveryTarget(str, Enum):
"""One member per existing table the projector writes; each is applied and receipted independently."""
KEY_SPEND = "LiteLLM_VerificationToken"
USER_SPEND = "LiteLLM_UserTable"
TEAM_SPEND = "LiteLLM_TeamTable"
TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership"
ORGANIZATION_SPEND = "LiteLLM_OrganizationTable"
ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership"
END_USER_SPEND = "LiteLLM_EndUserTable"
AGENT_SPEND = "LiteLLM_AgentsTable"
TAG_SPEND = "LiteLLM_TagTable"
PROJECT_SPEND = "LiteLLM_ProjectTable"
MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable"
BUDGET_SPEND = "LiteLLM_BudgetTable"
BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend"
DAILY_USER_SPEND = "LiteLLM_DailyUserSpend"
DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend"
DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend"
DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend"
DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend"
DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend"
DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage"
DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend"
AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend"
_SUBSET_PAIRS: Final = (
(MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS),
(MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS),
)
def _check_subsets(values: Mapping[str, Decimal]) -> None:
for part, whole in _SUBSET_PAIRS:
if part.value in values and whole.value in values and values[part.value] > values[whole.value]:
raise ValueError(f"{part.value} cannot exceed {whole.value}")
class Metric(BaseModel):
model_config = _CONFIG
name: MetricName
value: Quantity
unit: Literal["count", "USD"] = "count"
@model_validator(mode="after")
def validate_unit(self) -> Self:
if (self.name is MetricName.SPEND) != (self.unit == "USD"):
raise ValueError("SPEND requires USD; usage metrics require count")
if self.unit == "count" and self.value != self.value.to_integral_value():
raise ValueError("Count metrics require integral values")
return self
class Dimension(BaseModel):
model_config = _CONFIG
name: DimensionName
value: Identifier
_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP})
def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]:
if len(frozenset(values)) != len(values):
raise ValueError("Duplicate dimension values")
single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS)
if len(frozenset(single_names)) != len(single_names):
raise ValueError("Only tag and model access group dimensions may have multiple values")
return values
Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)]
_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS))
def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]:
if len(frozenset(value.name for value in values)) != len(values):
raise ValueError("Duplicate metric names")
return values
Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)]
def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None:
if end < start or (end == start and not allow_empty):
raise ValueError("End must follow start")
class InboundUsageEvent(BaseModel):
model_config = _CONFIG
event_id: Identifier
request_id: Identifier
call_type: CallTypes
started_at: Instant
ended_at: Instant
outcome: Outcome
dimensions: Dimensions
metrics: Metrics
@model_validator(mode="after")
def validate_caller_activity(self) -> Self:
_interval(self.started_at, self.ended_at, allow_empty=True)
if any(metric.name not in _CALLER_METRICS for metric in self.metrics):
raise ValueError("Inbound events own only caller request metrics")
if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics):
raise ValueError("Inbound events count exactly one caller request")
for metric in self.metrics:
if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED):
raise ValueError("Successful request count must match outcome")
if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED):
raise ValueError("Failed request count must match outcome")
return self
class OutboundUsageEvent(BaseModel):
model_config = _CONFIG
event_id: Identifier
attempt_id: Identifier
request_id: Identifier | None
retry_of_attempt_id: Identifier | None
call_type: CallTypes
started_at: Instant
ended_at: Instant
outcome: Outcome
dimensions: Dimensions
metrics: Metrics | None
@model_validator(mode="after")
def validate_attempt_usage(self) -> Self:
_interval(self.started_at, self.ended_at, allow_empty=True)
if self.retry_of_attempt_id == self.attempt_id:
raise ValueError("An attempt cannot retry itself")
if self.metrics is not None and any(
metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics
):
raise ValueError("Outbound events own only attempt usage metrics")
if self.metrics is not None:
_check_subsets({metric.name.value: metric.value for metric in self.metrics})
return self
PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"]
class Rate(BaseModel):
model_config = _CONFIG
measure: Identifier
unit: PricingUnit
usd_per_unit: Quantity
class PricedQuantity(BaseModel):
"""A pricing input in its billing unit. Only seconds may be fractional."""
model_config = _CONFIG
measure: Identifier
unit: PricingUnit
value: Quantity
@model_validator(mode="after")
def validate_discrete_quantity(self) -> Self:
if self.unit != "second" and self.value != self.value.to_integral_value():
raise ValueError("Discrete priced quantities require integral values")
return self
class Multiplier(BaseModel):
"""A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5."""
model_config = _CONFIG
name: Identifier
value: Identifier
factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)]
class NamedCost(BaseModel):
"""A free-form extra charge by name, e.g. azure_model_router_flat_cost."""
model_config = _CONFIG
name: Identifier
amount: Quantity
class PricingSnapshot(BaseModel):
"""Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
Component sums and float tolerances belong to reconciliation, not this structural contract.
"""
model_config = _CONFIG
schema_version: Literal[1] = 1
calculator_version: Identifier
model: Identifier
provider: Identifier | None
pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"]
priced_model_key: Identifier
multipliers: tuple[Multiplier, ...] = ()
rates: tuple[Rate, ...] = ()
usage: tuple[PricedQuantity, ...] = ()
input_cost: Quantity | None = None
output_cost: Quantity | None = None
cache_read_cost: Quantity | None = None
cache_creation_cost: Quantity | None = None
reasoning_cost: Quantity | None = None
tool_usage_cost: Quantity | None = None
guardrail_cost: Quantity | None = None
additional_costs: tuple[NamedCost, ...] = ()
original_cost: Quantity | None = None
discount_percent: Quantity | None = None
discount_amount: Quantity | None = None
margin_percent: Quantity | None = None
margin_fixed_amount: Quantity | None = None
margin_total_amount: Quantity | None = None
upstream_reported_cost: Quantity | None = None
total_cost: Quantity
@model_validator(mode="after")
def validate_snapshot(self) -> Self:
rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates)
quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage)
if len(rate_keys) != len(self.rates):
raise ValueError("Duplicate rate measures and units")
if len(quantity_keys) != len(self.usage):
raise ValueError("Duplicate priced quantity measures and units")
if not rate_keys.issubset(quantity_keys):
raise ValueError("Every rate requires one priced quantity with the same measure and unit")
if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs):
raise ValueError("Duplicate additional cost names")
if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers):
raise ValueError("Duplicate multiplier names")
_check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"})
return self
class CostEvent(BaseModel):
model_config = _CONFIG
event_id: Identifier
usage_event_id: Identifier
occurred_at: Instant
dimensions: Dimensions
charges: Metrics
snapshot: PricingSnapshot
@model_validator(mode="after")
def validate_charge(self) -> Self:
if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND:
raise ValueError("A cost event requires one authoritative USD spend metric")
if self.event_id == self.usage_event_id:
raise ValueError("Cost and usage events require distinct identities")
if self.snapshot.total_cost != self.charges[0].value:
raise ValueError("Snapshot total must equal the authoritative spend")
return self
class Unpriced(BaseModel):
model_config = _CONFIG
usage_event_id: Identifier
reason: Identifier
class Measurement(BaseModel):
model_config = _CONFIG
source_event_id: Identifier
occurred_at: Instant
dimensions: Dimensions
metric: Metric
class DimensionFilter(BaseModel):
model_config = _CONFIG
name: DimensionName
values: Annotated[tuple[Identifier, ...], Field(min_length=1)]
class AggregateQuery(BaseModel):
model_config = _CONFIG
metric: MetricName
filters: tuple[DimensionFilter, ...]
group_by: tuple[DimensionName, ...]
start: Instant
end: Instant
granularity: Granularity | None
@model_validator(mode="after")
def validate_query(self) -> Self:
_interval(self.start, self.end)
if len(frozenset(item.name for item in self.filters)) != len(self.filters):
raise ValueError("Use one IN filter per dimension; filters combine with AND")
if len(frozenset(self.group_by)) != len(self.group_by):
raise ValueError("Duplicate group-by dimensions")
if any(
(instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0)
for instant in (self.start, self.end)
):
raise ValueError("Initial reporting queries require UTC day boundaries")
return self
class AggregateRow(BaseModel):
model_config = _CONFIG
start: Instant
end: Instant
dimensions: Dimensions
value: Quantity
@model_validator(mode="after")
def validate_bounds(self) -> Self:
_interval(self.start, self.end)
return self
def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]:
"""Validate [start, end) buckets per group; overlapping tag groups are not additive."""
groups: Final = frozenset(frozenset(row.dimensions) for row in rows)
for group in groups:
ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start)
if any(left.end > right.start for left, right in zip(ordered, ordered[1:])):
raise ValueError("Aggregate buckets overlap within a dimension group")
return rows
class AdmissionDecision(BaseModel):
model_config = _CONFIG
allowed: bool
reason: Identifier | None = None

19
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -3,7 +3,6 @@ Handler for transforming /chat/completions api requests to litellm.responses req
"""
import json
import os
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from itertools import accumulate, chain
from types import MappingProxyType
@ -43,8 +42,8 @@ from litellm.responses.sse_output_recovery import (
)
from litellm.responses.utils import (
ResponsesAPIRequestUtils,
auto_reasoning_summary,
normalize_responses_api_stream_options,
opt_in_auto_reasoning_summary,
)
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
@ -1745,19 +1744,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
**cast(Reasoning, reasoning_effort) # cast-ok: dict is forwarded verbatim to the provider
)
# Check if auto-summary is enabled via flag or environment variable
# Priority: litellm.reasoning_auto_summary flag > LITELLM_REASONING_AUTO_SUMMARY env var
auto_summary_enabled: Final = (
litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
)
summary: Final = opt_in_auto_reasoning_summary(model, custom_llm_provider)
return (
Reasoning(
effort=reasoning_effort,
summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider),
)
if auto_summary_enabled
else Reasoning(effort=reasoning_effort)
Reasoning(effort=reasoning_effort)
if summary is None
else Reasoning(effort=reasoning_effort, summary=summary)
)
_map_reasoning_effort = map_reasoning_effort

2
litellm/constants.py
View file

@ -129,6 +129,7 @@ MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024
S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY"
S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
MAX_FILE_LIST_LIMIT: Final = 10000
ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20
DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10
@ -1096,6 +1097,7 @@ openai_compatible_providers: Final[list[str]] = [
"llamafile",
"lm_studio",
"galadriel",
"zerogpu",
"github_copilot", # GitHub Copilot Chat API
"chatgpt", # ChatGPT subscription API
"novita",

5
litellm/exceptions.py
View file

@ -317,6 +317,10 @@ class MalformedRequestFieldError(BadRequestError):
pass
class UnsupportedEndpointError(BadRequestError):
pass
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,
@ -1056,6 +1060,7 @@ LITELLM_EXCEPTION_TYPES: Final = [
BadRequestError,
UnprocessableEntityError,
UnsupportedParamsError,
UnsupportedEndpointError,
Timeout,
PermissionDeniedError,
RateLimitError,

4
litellm/images/main.py
View file

@ -1062,7 +1062,7 @@ def image_edit(
@client
async def aimage_edit(
image: FileTypes | list[FileTypes],
image: FileTypes | list[FileTypes] | None,
model: str,
prompt: str,
mask: str | None = None,
@ -1102,7 +1102,7 @@ async def aimage_edit(
model=model, api_base=local_vars.get("base_url", None)
)
images: Final = image if isinstance(image, list) else [image]
images: Final = image if isinstance(image, list) else ([image] if image is not None else [])
func: Final = partial(
image_edit,

2
litellm/integrations/otel/model/spans.py
View file

@ -338,6 +338,8 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
"LiteLLM_ToolTable",
"LiteLLM_TelemetryReport",
"LiteLLM_AccessGroupTable",
"LiteLLM_AccountingDelivery",
"LiteLLM_AccountingEvent",
"LiteLLM_ClaudeCodePluginTable",
"LiteLLM_MemoryTable",
"LiteLLM_AdaptiveRouterState",

6
litellm/interactions/__init__.py
View file

@ -32,11 +32,11 @@ Usage:
# Create a managed agent on the provider side
result = litellm.interactions.agents.create(
name="waverunner",
id="waverunner",
custom_llm_provider="gemini",
api_key="...",
base_agent="gemini-2.5-flash",
instructions="You are a helpful assistant.",
base_agent="antigravity-preview-09-2026",
system_instruction="You are a helpful assistant.",
)
Methods:

8
litellm/interactions/agents/__init__.py
View file

@ -3,11 +3,11 @@ litellm.interactions.agents
Full CRUD SDK for provider-side managed agents (e.g. Gemini v1beta/agents).
litellm.interactions.agents.create(name=..., ...)
litellm.interactions.agents.create(id=..., ...)
litellm.interactions.agents.list(api_key=...)
litellm.interactions.agents.get(name=..., ...)
litellm.interactions.agents.delete(name=..., ...)
litellm.interactions.agents.list_versions(name=..., ...)
litellm.interactions.agents.get(id=..., ...)
litellm.interactions.agents.delete(id=..., ...)
litellm.interactions.agents.list_versions(id=..., ...)
Async counterparts: acreate, alist, aget, adelete, alist_versions
"""

29
litellm/interactions/agents/http_handler.py
View file

@ -17,6 +17,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.agents import (
AgentCreateRequest,
AgentCreateResponse,
AgentDeleteResult,
AgentListResponse,
@ -35,7 +36,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
def create_agent(
self,
agents_api_config: BaseAgentsAPIConfig,
name: str,
create_request: AgentCreateRequest,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, str] | None = None,
@ -47,7 +48,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
if _is_async:
return self.async_create_agent(
agents_api_config=agents_api_config,
name=name,
create_request=create_request,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
@ -63,12 +64,13 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
api_base=litellm_params.get("api_base"),
litellm_params=dict(litellm_params),
)
data: Final = agents_api_config.transform_create_request(name=name, litellm_params=dict(litellm_params))
if extra_body:
data.update(extra_body)
data: Final = {
**agents_api_config.transform_create_request(create_request),
**(extra_body or {}),
}
logging_obj.pre_call(
input=name,
input=create_request.id,
api_key="",
additional_args={
"complete_input_dict": data,
@ -85,12 +87,12 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
original_response=response.text,
additional_args={"complete_input_dict": data},
)
return agents_api_config.transform_create_response(raw_response=response, name=name)
return agents_api_config.transform_create_response(raw_response=response, name=create_request.id)
async def async_create_agent(
self,
agents_api_config: BaseAgentsAPIConfig,
name: str,
create_request: AgentCreateRequest,
litellm_params: GenericLiteLLMParams,
logging_obj: LiteLLMLoggingObj,
extra_headers: dict[str, str] | None = None,
@ -106,12 +108,13 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
api_base=litellm_params.get("api_base"),
litellm_params=dict(litellm_params),
)
data: Final = agents_api_config.transform_create_request(name=name, litellm_params=dict(litellm_params))
if extra_body:
data.update(extra_body)
data: Final = {
**agents_api_config.transform_create_request(create_request),
**(extra_body or {}),
}
logging_obj.pre_call(
input=name,
input=create_request.id,
api_key="",
additional_args={
"complete_input_dict": data,
@ -130,7 +133,7 @@ class AgentsHTTPHandler(InteractionsHTTPHandler):
original_response=response.text,
additional_args={"complete_input_dict": data},
)
return agents_api_config.transform_create_response(raw_response=response, name=name)
return agents_api_config.transform_create_response(raw_response=response, name=create_request.id)
# ------------------------------------------------------------------ #
# LIST #

270
litellm/interactions/agents/main.py
View file

@ -6,41 +6,47 @@ Usage:
# Create
response = litellm.interactions.agents.create(
name="waverunner",
id="waverunner",
custom_llm_provider="gemini",
api_key="...",
base_agent="gemini-2.5-flash",
instructions="You are a helpful assistant.",
base_agent="antigravity-preview-09-2026",
system_instruction="You are a helpful assistant.",
agent_config={"type": "antigravity", "model": "gemini-3.8-flash"},
)
# List
response = litellm.interactions.agents.list(api_key="...", custom_llm_provider="gemini")
# Get
response = litellm.interactions.agents.get(name="waverunner", api_key="...")
response = litellm.interactions.agents.get(id="waverunner", api_key="...")
# Delete
result = litellm.interactions.agents.delete(name="waverunner", api_key="...")
result = litellm.interactions.agents.delete(id="waverunner", api_key="...")
# List versions
result = litellm.interactions.agents.list_versions(name="waverunner", api_key="...")
result = litellm.interactions.agents.list_versions(id="waverunner", api_key="...")
# Async versions: acreate, alist, aget, adelete, alist_versions
"""
import asyncio
import contextvars
from collections.abc import Coroutine, Mapping
from collections.abc import Coroutine, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import Any, Final
import httpx
from pydantic import ValidationError
import litellm
from litellm.interactions.agents.http_handler import agents_http_handler
from litellm.interactions.agents.utils import get_provider_agents_api_config
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.agents import (
AgentCreateRequest,
AgentCreateResponse,
AgentDeleteResult,
AgentListResponse,
@ -89,14 +95,120 @@ def _make_logging_obj(
return litellm_logging_obj
@dataclass(frozen=True, slots=True)
class _InvalidAgentRequest:
message: str
def _alias_conflict(field: str, value: str | None, alias: str, alias_value: str | None) -> str | None:
if value is None or alias_value is None or value == alias_value:
return None
return f"'{alias}' is an alias of '{field}', but the request gives them different values. Pass only '{field}'."
def _agent_id_or_error(id: str | None, name: str | None) -> str | _InvalidAgentRequest:
conflict: Final = _alias_conflict("id", id, "name", name)
if conflict is not None:
return _InvalidAgentRequest(conflict)
agent_id: Final = id if id is not None else name
if agent_id is None:
return _InvalidAgentRequest("Missing required parameter: 'id' (or its alias 'name').")
return agent_id
def _required_agent_id(id: str | None, name: str | None, llm_provider: str) -> str:
agent_id: Final = _agent_id_or_error(id, name)
if isinstance(agent_id, _InvalidAgentRequest):
raise litellm.BadRequestError(message=agent_id.message, model=id or name or "", llm_provider=llm_provider)
return agent_id
# ================================================================== #
# CREATE #
# ================================================================== #
_CREATE_FIELD_SHAPES: Final = MappingProxyType(
{
"id": "a string",
"base_agent": "a string",
"agent_config": "an object",
"system_instruction": "a string",
"base_environment": "an environment id string or an object",
"description": "a string",
"tools": "a list of objects",
}
)
def _invalid_create_fields_message(invalid_fields: frozenset[str], caller_names: Mapping[str, str]) -> str:
problems: Final = "; ".join(
f"'{caller_names.get(field, field)}' must be {_CREATE_FIELD_SHAPES[field]}" for field in sorted(invalid_fields)
)
drop_hint: Final = "" if "id" in invalid_fields else " Set drop_params to drop invalid optional fields."
return f"Invalid agent create request: {problems}.{drop_hint}"
def _validated_create_request(
fields: Mapping[str, object], drop_invalid_fields: bool, caller_names: Mapping[str, str]
) -> AgentCreateRequest | _InvalidAgentRequest:
try:
return AgentCreateRequest.model_validate(fields)
except ValidationError as e:
invalid_fields: Final = frozenset(str(error["loc"][0]) for error in e.errors())
if drop_invalid_fields and "id" not in invalid_fields:
return AgentCreateRequest.model_validate(
{field: value for field, value in fields.items() if field not in invalid_fields}
)
return _InvalidAgentRequest(_invalid_create_fields_message(invalid_fields, caller_names))
def _create_request_or_error(
id: str | None,
name: str | None,
base_agent: str | None,
agent_config: Mapping[str, object] | None,
system_instruction: str | None,
instructions: str | None,
base_environment: InteractionEnvironment | None,
description: str | None,
tools: Sequence[Mapping[str, object]] | None,
drop_invalid_fields: bool,
) -> AgentCreateRequest | _InvalidAgentRequest:
agent_id: Final = _agent_id_or_error(id, name)
if isinstance(agent_id, _InvalidAgentRequest):
return agent_id
instruction_conflict: Final = _alias_conflict(
"system_instruction", system_instruction, "instructions", instructions
)
if instruction_conflict is not None and not drop_invalid_fields:
return _InvalidAgentRequest(instruction_conflict)
sent_only_instructions: Final = system_instruction is None and instructions is not None
return _validated_create_request(
MappingProxyType(
{
"id": agent_id,
"base_agent": base_agent,
"agent_config": agent_config,
"system_instruction": instructions if sent_only_instructions else system_instruction,
"base_environment": base_environment,
"description": description,
"tools": tools,
}
),
drop_invalid_fields,
MappingProxyType(
{
"id": "name" if id is None else "id",
"system_instruction": "instructions" if sent_only_instructions else "system_instruction",
}
),
)
@client
async def acreate(
name: str,
id: str | None = None,
base_agent: str | None = None,
instructions: str | None = None,
base_environment: InteractionEnvironment | None = None,
@ -104,6 +216,12 @@ async def acreate(
extra_headers: dict[str, str] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
agent_config: Mapping[str, object] | None = None,
system_instruction: str | None = None,
description: str | None = None,
tools: Sequence[Mapping[str, object]] | None = None,
**kwargs,
) -> AgentCreateResponse:
"""Async: Create a managed agent on the provider side."""
@ -113,7 +231,7 @@ async def acreate(
kwargs["acreate_agent"] = True
func: Final = partial(
create,
name=name,
id=id,
base_agent=base_agent,
instructions=instructions,
base_environment=base_environment,
@ -121,6 +239,11 @@ async def acreate(
extra_headers=extra_headers,
extra_body=extra_body,
timeout=timeout,
name=name,
agent_config=agent_config,
system_instruction=system_instruction,
description=description,
tools=tools,
**kwargs,
)
ctx: Final = contextvars.copy_context()
@ -130,7 +253,7 @@ async def acreate(
return init_response
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider or "gemini",
original_exception=e,
completion_kwargs=local_vars,
@ -140,7 +263,7 @@ async def acreate(
@client
def create(
name: str,
id: str | None = None,
base_agent: str | None = None,
instructions: str | None = None,
base_environment: InteractionEnvironment | None = None,
@ -148,40 +271,61 @@ def create(
extra_headers: dict[str, str] | None = None,
extra_body: Mapping[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
agent_config: Mapping[str, object] | None = None,
system_instruction: str | None = None,
description: str | None = None,
tools: Sequence[Mapping[str, object]] | None = None,
**kwargs,
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
"""
Sync: Create a managed agent on the provider side.
Args:
name: Name for the agent (required).
base_agent: Base agent to derive from (e.g. "waverunner").
instructions: System instructions for the agent.
id: Identifier for the agent. Exactly one of ``id`` or its alias ``name`` is required.
base_agent: Base agent to derive from (e.g. "antigravity-preview-09-2026").
instructions: Alias of ``system_instruction``.
base_environment: Environment to fork from — an env_id string or a
dict like ``{"type": "remote", "sources": [...]}``.
custom_llm_provider: Provider to use, e.g. "gemini".
extra_headers: Additional HTTP headers.
extra_body: Additional request body fields.
timeout: Request timeout.
name: Alias of ``id``.
agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``.
system_instruction: System instruction for the agent.
description: Description of the agent for developers.
tools: Tools available to the agent, e.g. ``[{"type": "code_execution"}]``.
**kwargs: Forwarded to GenericLiteLLMParams (api_key, api_base, etc.).
"""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
custom_llm_provider = custom_llm_provider or "gemini"
try:
_is_async: Final = kwargs.pop("acreate_agent", False) is True
if base_agent is not None:
kwargs["base_agent"] = base_agent
if instructions is not None:
kwargs["instructions"] = instructions
if base_environment is not None:
kwargs["base_environment"] = base_environment
create_request: Final = _create_request_or_error(
id=id,
name=name,
base_agent=base_agent,
agent_config=agent_config,
system_instruction=system_instruction,
instructions=instructions,
base_environment=base_environment,
description=description,
tools=tools,
drop_invalid_fields=drops_unsupported_params(kwargs.get("drop_params")),
)
if isinstance(create_request, _InvalidAgentRequest):
raise litellm.BadRequestError(
message=create_request.message, model=id or name or "", llm_provider=custom_llm_provider
)
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
litellm_params: Final = GenericLiteLLMParams(**kwargs)
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "create_agent", {})
logging_obj: Final = _make_logging_obj(kwargs, create_request.id, custom_llm_provider, "create_agent", {})
config: Final = _get_agents_api_config(custom_llm_provider)
return agents_http_handler.create_agent(
agents_api_config=config,
name=name,
create_request=create_request,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
@ -191,7 +335,7 @@ def create(
)
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
@ -247,7 +391,7 @@ def list(
) -> AgentListResponse | Coroutine[object, object, AgentListResponse]:
"""Sync: List all agents on the provider side."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
custom_llm_provider = custom_llm_provider or "gemini"
try:
_is_async: Final = kwargs.pop("alist_agents", False) is True
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
@ -279,19 +423,22 @@ def list(
@client
async def aget(
name: str,
id: str | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
**kwargs,
) -> AgentCreateResponse:
"""Async: Get a specific agent by name."""
"""Async: Get a specific agent by id (``name`` is an alias of ``id``)."""
local_vars: Final = locals()
try:
loop: Final = asyncio.get_event_loop()
kwargs["aget_agent"] = True
func: Final = partial(
get,
id=id,
name=name,
custom_llm_provider=custom_llm_provider or "gemini",
extra_headers=extra_headers,
@ -305,7 +452,7 @@ async def aget(
return init_response
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider or "gemini",
original_exception=e,
completion_kwargs=local_vars,
@ -315,24 +462,27 @@ async def aget(
@client
def get(
name: str,
id: str | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
**kwargs,
) -> AgentCreateResponse | Coroutine[object, object, AgentCreateResponse]:
"""Sync: Get a specific agent by name."""
"""Sync: Get a specific agent by id (``name`` is an alias of ``id``)."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
custom_llm_provider = custom_llm_provider or "gemini"
try:
_is_async: Final = kwargs.pop("aget_agent", False) is True
agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
litellm_params: Final = GenericLiteLLMParams(**kwargs)
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "get_agent", {"name": name})
logging_obj: Final = _make_logging_obj(kwargs, agent_id, custom_llm_provider, "get_agent", {"name": agent_id})
config: Final = _get_agents_api_config(custom_llm_provider)
return agents_http_handler.get_agent(
agents_api_config=config,
name=name,
name=agent_id,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
@ -341,7 +491,7 @@ def get(
)
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
@ -356,19 +506,22 @@ def get(
@client
async def adelete(
name: str,
id: str | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
**kwargs,
) -> AgentDeleteResult:
"""Async: Delete a specific agent by name."""
"""Async: Delete a specific agent by id (``name`` is an alias of ``id``)."""
local_vars: Final = locals()
try:
loop: Final = asyncio.get_event_loop()
kwargs["adelete_agent"] = True
func: Final = partial(
delete,
id=id,
name=name,
custom_llm_provider=custom_llm_provider or "gemini",
extra_headers=extra_headers,
@ -382,7 +535,7 @@ async def adelete(
return init_response
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider or "gemini",
original_exception=e,
completion_kwargs=local_vars,
@ -392,24 +545,29 @@ async def adelete(
@client
def delete(
name: str,
id: str | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
**kwargs,
) -> AgentDeleteResult | Coroutine[object, object, AgentDeleteResult]:
"""Sync: Delete a specific agent by name."""
"""Sync: Delete a specific agent by id (``name`` is an alias of ``id``)."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
custom_llm_provider = custom_llm_provider or "gemini"
try:
_is_async: Final = kwargs.pop("adelete_agent", False) is True
agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
litellm_params: Final = GenericLiteLLMParams(**kwargs)
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "delete_agent", {"name": name})
logging_obj: Final = _make_logging_obj(
kwargs, agent_id, custom_llm_provider, "delete_agent", {"name": agent_id}
)
config: Final = _get_agents_api_config(custom_llm_provider)
return agents_http_handler.delete_agent(
agents_api_config=config,
name=name,
name=agent_id,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
@ -418,7 +576,7 @@ def delete(
)
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,
@ -433,19 +591,22 @@ def delete(
@client
async def alist_versions(
name: str,
id: str | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
**kwargs,
) -> AgentVersionsResponse:
"""Async: List versions of a specific agent."""
"""Async: List versions of a specific agent by id (``name`` is an alias of ``id``)."""
local_vars: Final = locals()
try:
loop: Final = asyncio.get_event_loop()
kwargs["alist_agent_versions"] = True
func: Final = partial(
list_versions,
id=id,
name=name,
custom_llm_provider=custom_llm_provider or "gemini",
extra_headers=extra_headers,
@ -459,7 +620,7 @@ async def alist_versions(
return init_response
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider or "gemini",
original_exception=e,
completion_kwargs=local_vars,
@ -469,24 +630,29 @@ async def alist_versions(
@client
def list_versions(
name: str,
id: str | None = None,
custom_llm_provider: str | None = None,
extra_headers: dict[str, str] | None = None,
timeout: float | httpx.Timeout | None = None,
*,
name: str | None = None,
**kwargs,
) -> AgentVersionsResponse | Coroutine[object, object, AgentVersionsResponse]:
"""Sync: List versions of a specific agent."""
"""Sync: List versions of a specific agent by id (``name`` is an alias of ``id``)."""
local_vars: Final = locals()
custom_llm_provider = custom_llm_provider or kwargs.get("custom_llm_provider") or "gemini"
custom_llm_provider = custom_llm_provider or "gemini"
try:
_is_async: Final = kwargs.pop("alist_agent_versions", False) is True
agent_id: Final = _required_agent_id(id, name, custom_llm_provider)
kwargs.setdefault("custom_llm_provider", custom_llm_provider)
litellm_params: Final = GenericLiteLLMParams(**kwargs)
logging_obj: Final = _make_logging_obj(kwargs, name, custom_llm_provider, "list_agent_versions", {"name": name})
logging_obj: Final = _make_logging_obj(
kwargs, agent_id, custom_llm_provider, "list_agent_versions", {"name": agent_id}
)
config: Final = _get_agents_api_config(custom_llm_provider)
return agents_http_handler.list_agent_versions(
agents_api_config=config,
name=name,
name=agent_id,
litellm_params=litellm_params,
logging_obj=logging_obj,
extra_headers=extra_headers,
@ -495,7 +661,7 @@ def list_versions(
)
except Exception as e:
raise litellm.exception_type(
model=name,
model=id or name or "",
custom_llm_provider=custom_llm_provider,
original_exception=e,
completion_kwargs=local_vars,

17
litellm/interactions/main.py
View file

@ -49,6 +49,7 @@ from litellm.interactions.utils import (
InteractionsAPIRequestUtils,
get_provider_interactions_api_config,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.interactions import (
CancelInteractionResult,
@ -100,6 +101,8 @@ async def acreate(
timeout: float | httpx.Timeout | None = None,
# LiteLLM params
custom_llm_provider: str | None = None,
*,
agent_config: Mapping[str, object] | None = None,
**kwargs,
) -> InteractionsAPIResponse | AsyncIterator[InteractionsAPIStreamingResponse]:
"""
@ -121,6 +124,7 @@ async def acreate(
string, or a config object such as
``{"type": "remote", "sources": [...]}`` /
``{"type": "remote", "network": {...}}``
agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``
response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
response_format: JSON schema for response format
response_mime_type: MIME type of the response
@ -155,6 +159,7 @@ async def acreate(
store=store,
background=background,
environment=environment,
agent_config=agent_config,
response_modalities=response_modalities,
response_format=response_format,
response_mime_type=response_mime_type,
@ -225,6 +230,8 @@ def create(
timeout: float | httpx.Timeout | None = None,
# LiteLLM params
custom_llm_provider: str | None = None,
*,
agent_config: Mapping[str, object] | None = None,
**kwargs,
) -> (
InteractionsAPIResponse
@ -250,6 +257,7 @@ def create(
string, or a config object such as
``{"type": "remote", "sources": [...]}`` /
``{"type": "remote", "network": {...}}``
agent_config: Agent runtime config, e.g. ``{"type": "antigravity", "model": "gemini-3.8-flash"}``
response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
response_format: JSON schema for response format
response_mime_type: MIME type of the response
@ -300,6 +308,15 @@ def create(
# Check if this is a bridge provider (litellm_responses) - similar to responses API
# Either provider is explicitly "litellm_responses" or no config found (bridge to responses)
if custom_llm_provider == "litellm_responses" or interactions_api_config is None:
if optional_params.get("agent_config") is not None and not drops_unsupported_params(
kwargs.get("drop_params")
):
raise litellm.UnsupportedParamsError(
message=f"'agent_config' is not supported for {custom_llm_provider} interactions. "
"Set drop_params to drop it.",
llm_provider=custom_llm_provider,
model=model,
)
# Bridge to litellm.responses() for non-native providers
from litellm.interactions.litellm_responses_transformation.handler import (
LiteLLMResponsesInteractionsHandler,

2
litellm/litellm_core_utils/README.md
View file

@ -11,4 +11,4 @@ Core files:
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.

43
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -2,8 +2,12 @@
Utils used for litellm.transcription() and litellm.atranscription()
"""
import base64
import binascii
import hashlib
import math
import os
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Final
@ -331,6 +335,45 @@ def calculate_request_duration(file: FileTypes) -> float | None:
return None
DEFAULT_AUDIO_TOKEN_ESTIMATE: Final = 300
AUDIO_TOKENS_PER_SECOND: Final = 10
AUDIO_BYTES_PER_TOKEN: Final = 1600
_MAX_AUDIO_ESTIMATE_BYTES: Final = 16 * 1024 * 1024
def _audio_token_rate(model: str) -> int | None:
if model.startswith(
("gemini-", "gemini/gemini-", "vertex_ai/gemini-", "vertex_ai_beta/gemini-", "openrouter/google/gemini-")
):
family: Final = model.rsplit("/", 1)[-1]
return 25 if family.startswith(("gemini-3-", "gemini-3.")) else 32
if model.startswith(("gpt-", "openai/gpt-", "azure/gpt-", "openrouter/openai/gpt-")):
return AUDIO_TOKENS_PER_SECOND
return None
def estimate_audio_tokens(data: str | None, models: Sequence[str] = (), fallback_model: str | None = None) -> int:
"""Estimate audio tokens from container duration within a bounded decoding budget."""
if not data:
return DEFAULT_AUDIO_TOKEN_ESTIMATE
fallback_estimate: Final = max(len(data) * 3 // 4 // AUDIO_BYTES_PER_TOKEN, DEFAULT_AUDIO_TOKEN_ESTIMATE)
padding: Final = 2 if data.endswith("==") else 1 if data.endswith("=") else 0
if len(data) * 3 // 4 - padding > _MAX_AUDIO_ESTIMATE_BYTES:
return fallback_estimate
try:
duration: Final = calculate_request_duration(base64.b64decode(data, validate=True))
except (ValueError, binascii.Error):
return fallback_estimate
if duration is None or not math.isfinite(duration) or duration <= 0:
return fallback_estimate
fallback_rate: Final = _audio_token_rate(fallback_model) if fallback_model is not None else None
rate: Final = max(
(_audio_token_rate(model) or fallback_rate or AUDIO_TOKENS_PER_SECOND for model in models),
default=fallback_rate or AUDIO_TOKENS_PER_SECOND,
)
return max(1, math.ceil(duration * rate))
DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg"

6
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -353,9 +353,9 @@ def is_cli_token_fresh(
buffer_hours: float = CLI_TOKEN_FRESHNESS_BUFFER_SECONDS / 3600,
) -> bool:
"""Check whether a cached CLI token is still within its expiration window.
Used by `lite auth print-token` to fail fast, without a network round trip,
once the cached token is past `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce`
credential carries its own `expires_at`, which is authoritative when present."""
Used by `lite auth print-token` to skip asking the proxy while the cached token
is younger than `LITELLM_CLI_JWT_EXPIRATION_HOURS`. A `--pkce` credential
carries its own `expires_at`, which is authoritative when present."""
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
expires_at: Final = (

1
litellm/litellm_core_utils/core_helpers.py
View file

@ -266,6 +266,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
"tool_use": "tool_calls",
"refusal": "content_filter",
"compaction": "length",
"model_context_window_exceeded": "length",
# Cohere
"COMPLETE": "stop",
"ERROR_TOXIC": "content_filter",

9
litellm/litellm_core_utils/litellm_logging.py
View file

@ -842,6 +842,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._async_success_enqueued: bool = False
self._sync_success_scheduled: bool = False
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
self._stream_failure_claimed: bool = False
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
def copy_for_new_call(self, litellm_call_id: str, start_time: datetime.datetime) -> "Logging":
@ -2561,6 +2562,12 @@ class Logging(LiteLLMLoggingBaseClass):
},
)
def claim_stream_failure_report(self) -> bool:
if self._stream_failure_claimed:
return False
self._stream_failure_claimed = True
return True
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
@ -2751,7 +2758,7 @@ class Logging(LiteLLMLoggingBaseClass):
if entry is None or not is_str_keyed_dict(litellm_params):
return
metadata: Final[object] = litellm_params.get("metadata")
bucket: Final = metadata if is_str_keyed_dict(metadata) else {}
bucket: Final = {**metadata} if is_str_keyed_dict(metadata) else {}
litellm_params["metadata"] = bucket
with_guardrail_entry(bucket, entry)

155
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -9,6 +9,7 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from graphlib import CycleError, TopologicalSorter
from itertools import chain, groupby, islice
from os import PathLike
@ -236,6 +237,160 @@ _IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image
_IMAGE_SCAN_MAX_DEPTH: Final = 4
@dataclass(frozen=True, slots=True)
class _TPMMediaProjection:
content: object
media: tuple[Mapping[str, object], ...] = ()
def _tpm_image_part(url: str, detail: object = "auto") -> Mapping[str, object]:
return {"type": "image_url", "image_url": {"url": url, "detail": detail}}
def _tpm_blank_fields(fields: Mapping[str, object], names: tuple[str, ...]) -> Mapping[str, object]:
return {key: "" if key in names and isinstance(value, str) else value for key, value in fields.items()}
def _tpm_file_media(part: Mapping[str, object]) -> _TPMMediaProjection:
from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
nested: Final = part.get("type") == "file"
fields: Final = as_str_mapping(part.get("file")) if nested else part
if fields is None:
return _TPMMediaProjection(part)
raw_data: Final = fields.get("file_data")
data: Final = raw_data if isinstance(raw_data, str) else ""
declared: Final = next(
(
value
for key in ("format", "mime_type", "content_type")
if isinstance(value := fields.get(key), str) and value
),
"",
)
mime: Final = (declared or parse_mime_type(data[:256]) or "").split(";", 1)[0].strip().lower()
if not mime.startswith(("image/", "audio/")):
return _TPMMediaProjection(part)
raw_url: Final = fields.get("file_url") or fields.get("file_id")
url: Final = data or (raw_url if isinstance(raw_url, str) else "")
header_end: Final = data.find(",", 0, 256)
encoded: Final = (
data[header_end + 1 :]
if header_end >= 0 and data[:header_end].endswith(";base64")
else ""
if data.startswith(("data:", "https://", "http://", "gs://"))
else data
)
media: Final = (
_tpm_image_part(
url
if not data or url.startswith(("data:", "https://", "http://", "gs://"))
else f"data:{mime};base64,{url}",
fields.get("detail", "auto"),
)
if mime.startswith("image/")
else {"type": "input_audio", "input_audio": {"data": encoded, "format": mime.removeprefix("audio/")}}
)
projected: Final = _tpm_blank_fields(fields, ("file_data", "file_url"))
return _TPMMediaProjection({**part, "file": projected} if nested else projected, (media,))
def _tpm_media_leaf(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is None:
return _TPMMediaProjection(value)
kind: Final = part.get("type")
if kind in ("file", "input_file"):
return _tpm_file_media(part)
if kind == "input_audio":
audio: Final = as_str_mapping(part.get("input_audio"))
return _TPMMediaProjection(
{**part, "input_audio": _tpm_blank_fields(audio, ("data",))} if audio is not None else value,
({"type": "input_audio", "input_audio": part.get("input_audio")},),
)
if kind in ("image_url", "input_image", "image", "computer_screenshot"):
image: Final = as_str_mapping(part.get("image_url"))
source: Final = as_str_mapping(part.get("source"))
canonical: Final = _tpm_image_part(
_image_part_url({**part, "type": "input_image"} if kind == "computer_screenshot" else part) or "",
(image or part).get("detail", part.get("detail", "auto")),
)
projected: Final = (
{**part, "image_url": _tpm_blank_fields(image, ("url",))}
if image is not None and kind in ("image_url", "input_image", "computer_screenshot")
else {**part, "source": _tpm_blank_fields(source, ("data", "url"))}
if source is not None and kind == "image" and source.get("type") in ("base64", "url")
else _tpm_blank_fields(part, ("image_url", "url"))
)
return _TPMMediaProjection(projected, (canonical,))
return _TPMMediaProjection(value)
def _tpm_media_list(value: object, split: Callable[[object], _TPMMediaProjection]) -> _TPMMediaProjection:
if not isinstance(value, (list, tuple)):
return _TPMMediaProjection(value)
parts: Final = cast(Sequence[object], value) # cast-ok: finite JSON content arrays
projected: Final = tuple(map(split, parts))
media: Final = tuple(chain.from_iterable(part.media for part in projected))
content: Final = tuple(part.content for part in projected)
return _TPMMediaProjection(parts if not media else content if isinstance(value, tuple) else list(content), media)
def _tpm_replace_field(part: Mapping[str, object], key: str, projection: _TPMMediaProjection) -> _TPMMediaProjection:
return _TPMMediaProjection({**part, key: projection.content} if projection.media else part, projection.media)
def _tpm_document_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
source: Final = as_str_mapping(part.get("source")) if part is not None else None
if part is None or part.get("type") != "document" or source is None or source.get("type") != "content":
return _tpm_media_leaf(value)
content: Final = _tpm_media_list(source.get("content"), _tpm_media_leaf)
return _tpm_replace_field(part, "source", _tpm_replace_field(source, "content", content))
def _tpm_tool_result_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is None or part.get("type") != "tool_result":
return _tpm_document_media(value)
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_document_media))
def _tpm_code_output_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
url: Final = part.get("url") if part is not None else None
if part is None or part.get("type") != "image" or not isinstance(url, str) or not url:
return _TPMMediaProjection(value)
return _TPMMediaProjection(_tpm_blank_fields(part, ("url",)), (_tpm_image_part(url),))
def _tpm_input_media(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is None:
return _TPMMediaProjection(value)
kind: Final = part.get("type")
if kind in ("function_call_output", "custom_tool_call_output"):
return _tpm_replace_field(part, "output", _tpm_media_list(part.get("output"), _tpm_tool_result_media))
if kind == "computer_call_output":
return _tpm_replace_field(part, "output", _tpm_media_leaf(part.get("output")))
if kind == "code_interpreter_call":
return _tpm_replace_field(part, "outputs", _tpm_media_list(part.get("outputs"), _tpm_code_output_media))
result: Final = part.get("result")
if kind == "image_generation_call" and isinstance(result, str) and result:
return _TPMMediaProjection(
_tpm_blank_fields(part, ("result",)),
(_tpm_image_part("data:application/octet-stream;base64," + result),),
)
if kind == "message" or (kind is None and isinstance(part.get("role"), str)):
return _tpm_replace_field(part, "content", _tpm_media_list(part.get("content"), _tpm_tool_result_media))
return _tpm_tool_result_media(value)
def split_media_for_tpm(value: object) -> tuple[object, tuple[Mapping[str, object], ...]]:
projection: Final = _tpm_media_list(value, _tpm_input_media)
return projection.content, projection.media
def _content_parts_contain_image(parts: Sequence[object]) -> bool:
"""Depth-bounded frontier walk over nested content lists, iterative because the repo bans
recursion; an Anthropic tool_result nests its image parts exactly one level down."""

92
litellm/litellm_core_utils/single_flight.py Normal file
View file

@ -0,0 +1,92 @@
import threading
from collections.abc import Callable
from concurrent.futures import Future
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Final, Generic, ParamSpec, TypeVar
_P: Final = ParamSpec("_P")
_R: Final = TypeVar("_R")
@dataclass(slots=True)
class SingleFlight(Generic[_R]):
lock: threading.Lock = field(default_factory=threading.Lock)
in_flight: Future[_R] | None = None
flights_started: int = 0
last_flight_failed: bool = False
def run(self, fetch: Callable[[], _R], while_busy: _R | None = None) -> _R:
leader: Final[Future[_R]] = Future()
with self.lock:
in_flight: Final = self.in_flight
joined_flight: Final = self.flights_started
retry_if_it_fails: Final = not self.last_flight_failed
if in_flight is None:
self._take_off(leader)
if in_flight is None:
return self._lead(leader, fetch)
if while_busy is not None:
return while_busy
try:
return in_flight.result()
except Exception:
retry: Final = self._claim_retry(joined_flight) if retry_if_it_fails else None
if retry is None:
raise
retry_flight, leads_retry = retry
return self._lead(retry_flight, fetch) if leads_retry else retry_flight.result()
def _take_off(self, leader: Future[_R]) -> None:
self.in_flight = leader
self.flights_started += 1
def _claim_retry(self, failed_flight: int) -> tuple[Future[_R], bool] | None:
leader: Final[Future[_R]] = Future()
with self.lock:
in_flight: Final = self.in_flight
if in_flight is not None:
return in_flight, False
if self.flights_started > failed_flight and self.last_flight_failed:
return None
self._take_off(leader)
return leader, True
def _lead(self, flight: Future[_R], fetch: Callable[[], _R]) -> _R:
try:
result: Final = fetch()
except BaseException as error:
self._land(failed=True)
flight.set_exception(error)
raise
self._land(failed=False)
flight.set_result(result)
return result
def _land(self, failed: bool) -> None:
with self.lock:
self.in_flight = None
self.last_flight_failed = failed
@dataclass(frozen=True, slots=True)
class SingleFlightCache(Generic[_P, _R]):
cached: Callable[..., _R]
clear: Callable[[], None]
lock: threading.Lock = field(default_factory=threading.Lock)
def __call__(
self,
*args: _P.args,
**kwargs: _P.kwargs, # kwargs-ok: forwarded verbatim to the cached builder, typed by its own signature
) -> _R:
with self.lock:
return self.cached(*args, **kwargs)
def cache_clear(self) -> None:
self.clear()
def single_flight_lru_cache(build: Callable[_P, _R]) -> SingleFlightCache[_P, _R]:
cached: Final = lru_cache(maxsize=128)(build)
return SingleFlightCache(cached, cached.cache_clear)

6
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1557,6 +1557,12 @@ class CustomStreamWrapper:
else: # openai / azure chat model
if isinstance(chunk, ChatCompletionChunk):
self._openai_handler_receives_chat_chunks = True
if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
raise litellm.InternalServerError(
message="Upstream generation aborted (finish_reason=abort)",
model=chunk.model,
llm_provider=self.custom_llm_provider,
)
if self.custom_llm_provider in [
LlmProviders.AZURE.value,
LlmProviders.AZURE_AI.value,

232
litellm/litellm_core_utils/token_counter.py
View file

@ -3,9 +3,12 @@
import base64
import io
import math
import re
import struct
from bisect import bisect_right
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, cast
import anyio
@ -35,6 +38,7 @@ from litellm.constants import (
TOKEN_COUNTER_MAX_EXACT_CHARS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
ChatTemplateToolSection,
chat_template_tool_section,
@ -188,22 +192,23 @@ def resize_image_high_res(
# Test the function with the given example
def calculate_tiles_needed(
resized_width,
resized_height,
tile_width=MAX_TILE_WIDTH,
tile_height=MAX_TILE_HEIGHT,
):
resized_width: int,
resized_height: int,
tile_width: int = MAX_TILE_WIDTH,
tile_height: int = MAX_TILE_HEIGHT,
) -> int:
tiles_across: Final = (resized_width + tile_width - 1) // tile_width
tiles_down: Final = (resized_height + tile_height - 1) // tile_height
total_tiles: Final = tiles_across * tiles_down
return total_tiles
def high_detail_image_token_upper_bound(base_tokens: int = 85) -> int:
def high_detail_image_token_upper_bound(base_tokens: int = 85, tile_tokens: int | None = None) -> int:
largest_tile_count: Final = calculate_tiles_needed(
MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES, MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES
)
return base_tokens + (base_tokens * 2) * largest_tile_count
per_tile: Final = base_tokens * 2 if tile_tokens is None else tile_tokens
return base_tokens + per_tile * largest_tile_count
def _unpack_ints(fmt: str, buffer: bytes) -> tuple[int, ...]:
@ -381,6 +386,219 @@ def calculate_img_tokens(
return total_tokens
_IMAGE_HEADER_BASE64_CHARS: Final = 65_536
_MAX_IMAGE_PATCHES: Final = 30_000
_ANTHROPIC_IMAGE_LIMITS: Final = MappingProxyType(
{"claude-standard": (ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX, 1568), "claude-high": (2576, 4784)}
)
_TILE_IMAGE_TOKEN_COSTS: Final = MappingProxyType(
{
"gpt-4o": (85, 170),
"gpt-4.1": (85, 170),
"gpt-4o-mini": (2833, 5667),
"gpt-5": (70, 140),
"gpt-5.1": (70, 140),
"o1": (75, 150),
"o1-pro": (75, 150),
"o3": (75, 150),
}
)
_PATCH_IMAGE_MULTIPLIERS: Final = MappingProxyType(
{
"gpt-6-astra": 1.2,
"gpt-5.6-sol": 1.2,
"gpt-5.6-terra": 1.2,
"gpt-5.6-luna": 1.2,
"gpt-5.5": 1.2,
"gpt-5.4": 1.2,
"gpt-5.4-mini": 1.2,
"gpt-5.4-nano": 1.2,
"gpt-5.2": 1.2,
"gpt-4.1-mini": 1.62,
}
)
def _inline_image_dimensions(data: str) -> tuple[int, int] | None:
separator: Final = data.find(",", 0, 256)
if not data.startswith("data:") or separator < 0 or not data[:separator].endswith(";base64"):
return None
encoded_header: Final = data[separator + 1 : separator + 1 + _IMAGE_HEADER_BASE64_CHARS]
try:
header: Final = base64.b64decode(encoded_header, validate=True)
image_type: Final = get_image_type(header)
if image_type not in ("png", "gif", "jpeg", "webp"):
return None
if image_type == "webp" and header[12:16] not in (b"VP8X", b"VP8 ", b"VP8L"):
return None
dimensions: Final = _image_dimensions_from_bytes(header)
except (ValueError, TypeError, struct.error):
return None
width, height = dimensions
return (width, height) if width > 0 and height > 0 else None
def _patch_image_limits(model: str, detail: str) -> tuple[int, int | None] | None:
if model in ("gpt-5.2", "gpt-4.1-mini"):
return None if detail == "original" else (2048, 6144)
original_default: Final = model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5")
effective_detail: Final = ("original" if original_default else "high") if detail == "auto" else detail
if effective_detail == "low":
return (2048, 6144) if model in ("gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano") else (512, None)
if effective_detail == "high":
return (65_535 if model == "gpt-6-astra" else 2048, 2500)
return (
(65_535, None) if model in ("gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna") else (6000, 10_000)
)
def _image_patch_count(width: int, height: int, max_dimension: int, budget: int | None) -> int:
dimension_scale: Final = min(1.0, max_dimension / max(width, height))
fitted_width: Final = max(1, round(width * dimension_scale))
fitted_height: Final = max(1, round(height * dimension_scale))
patches: Final = math.ceil(fitted_width / 32) * math.ceil(fitted_height / 32)
if budget is None or patches <= budget:
return patches
shrink: Final = math.sqrt(32**2 * budget / (fitted_width * fitted_height))
width_patches: Final = fitted_width * shrink / 32
height_patches: Final = fitted_height * shrink / 32
aligned_shrink: Final = shrink * min(
math.floor(width_patches) / width_patches, math.floor(height_patches) / height_patches
)
resized_width: Final = max(1, math.floor(fitted_width * aligned_shrink))
resized_height: Final = max(1, math.floor(fitted_height * aligned_shrink))
return math.ceil(resized_width / 32) * math.ceil(resized_height / 32)
def _image_policy_model_name(model: str) -> str | None:
provider, separator, provider_model = model.partition("/")
if not separator:
return model
if provider == "openai":
return provider_model
if provider == "openrouter":
return provider_model.removeprefix("openai/") if provider_model.startswith("openai/") else None
if provider not in ("azure", "azure_ai"):
return None
catalog_entry: Final = as_str_mapping(
cast( # cast-ok: read-only view of the legacy string-keyed registry; as_str_mapping checks the entry
Mapping[str, object], litellm.model_cost
).get(model)
)
if catalog_entry is not None and catalog_entry.get("litellm_provider") in ("azure", "azure_ai"):
return provider_model.rsplit("/", 1)[-1]
return provider_model.removeprefix("responses/").removeprefix("o_series/").removeprefix("openai/")
def _anthropic_image_family(model: str) -> str | None:
from litellm.utils import get_potential_model_names
provider, separator, _ = model.partition("/")
if separator and provider not in (
"anthropic",
"bedrock",
"bedrock_converse",
"bedrock_mantle",
"vertex_ai",
"vertex_ai_beta",
"azure_ai",
"openrouter",
):
return None
candidates: Final = get_potential_model_names(model, provider if separator else "anthropic")
catalog: Final = cast(Mapping[str, object], litellm.model_cost) # cast-ok: read-only view of the legacy registry
for candidate in dict.fromkeys(candidates.values()):
if not isinstance(candidate, str):
continue
if (entry := as_str_mapping(catalog.get(candidate))) is None or entry.get("supports_vision") is not True:
continue
if (
version := re.search(
r"(?:^|[./])claude-(?:(?:haiku|sonnet|opus)-)?(\d+)(?:[-.](\d{1,2})(?!\d))?", candidate
)
) is None:
continue
major, minor = int(version[1]), int(version[2] or 0)
return "claude-high" if (major, minor) >= (4, 7) else "claude-standard"
return None
def _anthropic_image_tokens(dimensions: tuple[int, int] | None, limits: tuple[int, int]) -> int:
max_edge, token_budget = limits
if dimensions is None:
return token_budget
long_edge, short_edge = max(dimensions), min(dimensions)
aspect_ratio: Final = long_edge / short_edge
def tokens_at_edge(edge: int) -> int:
return math.ceil(edge / 28) * math.ceil(max(round(edge / aspect_ratio), 1) / 28)
fitted_edge: Final = bisect_right(range(1, min(long_edge, max_edge) + 1), token_budget, key=tokens_at_edge)
return tokens_at_edge(max(fitted_edge, 1))
def _image_model_family(model: str) -> str | None:
provider_model: Final = _image_policy_model_name(model)
if provider_model is None:
return _anthropic_image_family(model)
family: Final = re.sub(r"-\d{4}-\d{2}-\d{2}$", "", provider_model)
canonical: Final = litellm.azure_llms.get(f"azure/{family}", family) if model.startswith("azure/") else family
return (
canonical
if canonical in _TILE_IMAGE_TOKEN_COSTS or canonical in _PATCH_IMAGE_MULTIPLIERS
else _anthropic_image_family(model)
)
def _image_policy_tokens(family: str | None, dimensions: tuple[int, int] | None, detail: str) -> int:
if family is None:
return DEFAULT_IMAGE_TOKEN_COUNT
anthropic_limits: Final = _ANTHROPIC_IMAGE_LIMITS.get(family)
if anthropic_limits is not None:
return _anthropic_image_tokens(dimensions, anthropic_limits)
tile_costs: Final = _TILE_IMAGE_TOKEN_COSTS.get(family)
multiplier: Final = _PATCH_IMAGE_MULTIPLIERS.get(family)
if tile_costs is not None:
base_tokens, tile_tokens = tile_costs
if detail == "original":
return DEFAULT_IMAGE_TOKEN_COUNT
if detail == "low":
return base_tokens
if dimensions is None:
return high_detail_image_token_upper_bound(base_tokens, tile_tokens)
width, height = dimensions
scale: Final = min(1.0, 2048 / max(width, height), 768 / min(width, height))
tiles: Final = calculate_tiles_needed(max(1, math.floor(width * scale)), max(1, math.floor(height * scale)))
return base_tokens + tiles * tile_tokens
limits: Final = _patch_image_limits(family, detail)
if limits is None or multiplier is None:
return DEFAULT_IMAGE_TOKEN_COUNT
max_dimension, patch_budget = limits
patches: Final = (
min(math.ceil(max_dimension / 32) ** 2, patch_budget if patch_budget is not None else _MAX_IMAGE_PATCHES)
if dimensions is None
else _image_patch_count(*dimensions, max_dimension, patch_budget)
)
return math.ceil(patches * multiplier)
def estimate_image_tokens(
model: str | Sequence[str], data: str, detail: str = "auto", *, fallback_model: str | None = None
) -> int:
"""Estimate image policies from at most 48 KiB of inline headers, without fetching media."""
if detail not in ("low", "high", "auto", "original"):
return DEFAULT_IMAGE_TOKEN_COUNT
models: Final = (model,) if isinstance(model, str) else model
fallback_family: Final = _image_model_family(fallback_model) if fallback_model is not None else None
families: Final = tuple(
dict.fromkeys(_image_model_family(candidate) or fallback_family for candidate in models or ("",))
)
if all(family is None for family in families):
return DEFAULT_IMAGE_TOKEN_COUNT
dimensions: Final = _inline_image_dimensions(data)
return max(_image_policy_tokens(family, dimensions, detail) for family in families)
TokenCounterFunction = Callable[[str], int]
"""
Type for a function that counts tokens in a string.

2
litellm/llms/aiohttp_openai/chat/transformation.py
View file

@ -82,4 +82,6 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
model_response.model = _json_response.get("model")
model_response.object = _json_response.get("object")
model_response.system_fingerprint = _json_response.get("system_fingerprint")
if "service_tier" in _json_response:
return model_response.model_copy(update={"service_tier": _json_response.get("service_tier")})
return model_response

59
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -161,11 +161,58 @@ def _has_non_chat_user_block(message: Mapping[str, object]) -> bool:
return any(isinstance(block, dict) and block.get("type") not in ValidUserMessageContentTypes for block in blocks)
_OPENAI_RESPONSES_BUILT_IN_TOOL_TYPES: Final = frozenset(
(
"apply_patch",
"code_interpreter",
"computer",
"computer_use_preview",
"file_search",
"image_generation",
"local_shell",
"mcp",
"shell",
"web_search",
"web_search_2025_08_26",
"web_search_preview",
"web_search_preview_2025_03_11",
)
)
_CHAT_INFERENCE_TRANSLATED_TOOL_TYPES: Final = frozenset(("namespace", "tool_search"))
def _is_responses_built_in_tool(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return isinstance(tool_type, str) and tool_type in _OPENAI_RESPONSES_BUILT_IN_TOOL_TYPES
def _is_chat_inference_translated_tool(tool: Mapping[str, object]) -> bool:
tool_type: Final = tool.get("type")
return isinstance(tool_type, str) and tool_type in _CHAT_INFERENCE_TRANSLATED_TOOL_TYPES
def _needs_tool_translation(tool: Mapping[str, object]) -> bool:
return _is_chat_tool(tool) or _is_chat_inference_translated_tool(tool) or _is_responses_built_in_tool(tool)
def _gets_placeholder_name(tool: Mapping[str, object]) -> bool:
from litellm.llms.anthropic.messages.adapters.transformation import is_provider_native_tool_dict
name: Final = tool.get("name")
return not (_is_chat_tool(tool) or is_provider_native_tool_dict(tool) or (isinstance(name, str) and name.strip()))
return not (
_is_chat_tool(tool)
or _is_chat_inference_translated_tool(tool)
or is_provider_native_tool_dict(tool)
or (isinstance(name, str) and name.strip())
)
def _chat_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_tools_for
return (tool,) if _is_chat_inference_translated_tool(tool) else chat_completion_tools_for(tool)
def _untranslatable_input(
@ -182,8 +229,8 @@ def _untranslatable_input(
unnamed_tool: Final = next(filter(_gets_placeholder_name, tools), None)
if unnamed_tool is not None:
return ChatCountTranslationError(
message=f"Invalid request for token counting: tool type {unnamed_tool.get('type')!r} "
"has no Chat Completions equivalent"
message=f"Invalid request for token counting: LiteLLM cannot count a tool of type "
f"{unnamed_tool.get('type')!r} for this model"
)
return None
@ -223,9 +270,7 @@ def _translated_chat_request(
chat_params: Mapping[str, object],
translator: _ChatRequestTranslator,
) -> _ChatParams:
from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_tools_for
chat_tools: Final = [dict(chat_tool) for chat_tool in chain.from_iterable(map(chat_completion_tools_for, tools))]
chat_tools: Final = [dict(chat_tool) for chat_tool in chain.from_iterable(map(_chat_tools_for, tools))]
openai_params: Final = {name: value for name, value in chat_params.items() if name != "drop_params"}
mapped_params: Final = {**openai_params, **({"tools": chat_tools} if chat_tools else {})}
chat_messages: Final = cast( # cast-ok: chat-shaped request messages, read the way chat inference reads them
@ -255,7 +300,7 @@ def anthropic_fields_for_chat_tool_use(
counted_tools: Final = tools or ()
if (
not chat_params
and not any(map(_is_chat_tool, counted_tools))
and not any(map(_needs_tool_translation, counted_tools))
and not any(map(_needs_chat_translation, messages))
):
return None

65
litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py
View file

@ -2,15 +2,16 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Iterator, Mapping
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final
import anyio
from pydantic import ConfigDict, field_validator
from litellm import verbose_logger
from litellm._logging import redact_internal_details_from_client_message
from litellm._uuid import uuid
from litellm.exceptions import MidStreamFallbackError
from litellm.exceptions import APIError, MidStreamFallbackError
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
@ -22,6 +23,7 @@ from litellm.llms.anthropic.messages.native.utils import (
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.responses.streaming_iterator import (
map_stream_error_to_exception,
partial_responses_incomplete_event_on_disconnect,
stream_error_status_and_message,
)
@ -130,10 +132,12 @@ class AnthropicResponsesStreamWrapper:
) -> None:
self.responses_stream = responses_stream
self.model = model
self.litellm_logging_obj = litellm_logging_obj
self.safeguards_evaluator: Final = safeguards_evaluator
self._message_id: str = f"msg_{uuid.uuid4()}"
if litellm_logging_obj is not None:
litellm_logging_obj.record_streamed_anthropic_message_id(self._message_id)
self._unreported_failure: Exception | None = None
self._current_block_index: int = -1
# Map item_id -> content_block_index so we can stop the right block later
self._item_id_to_block_index: dict[str, int] = {}
@ -377,16 +381,24 @@ class AnthropicResponsesStreamWrapper:
if event_type == "response.failed":
failed: Final = _FailedResponseEvent.model_validate(event)
status_code, message = stream_error_status_and_message(
failed.response.error if failed.response is not None else None
)
upstream_error: Final = failed.response.error if failed.response is not None else None
status_code, message = stream_error_status_and_message(upstream_error)
verbose_logger.error(
"AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed (%s): %s",
self.model,
status_code,
message,
)
self._fail_stream(status_code, message)
self._fail_stream(
status_code,
message,
failure=map_stream_error_to_exception(
upstream_error,
self.model,
getattr(self.responses_stream, "custom_llm_provider", None) or "",
logging_obj=self.litellm_logging_obj,
),
)
return
# ---- response completed -> message_delta + message_stop ----
@ -439,10 +451,36 @@ class AnthropicResponsesStreamWrapper:
self._sent_message_stop = True
return
def _fail_stream(self, status_code: int, message: str) -> None:
def _fail_stream(self, status_code: int, message: str, failure: Exception | None = None) -> None:
self._stream_failed = True
self._unreported_failure = failure or APIError(
status_code=status_code,
message=message,
llm_provider=getattr(self.responses_stream, "custom_llm_provider", None),
model=self.model,
)
self._chunk_queue.append(_anthropic_error_chunk(status_code, message))
async def _report_unreported_failure(self) -> None:
failure: Final = self._unreported_failure
if failure is None:
return
self._unreported_failure = None
on_failure: Final[Callable[[Exception], Awaitable[None]] | None] = getattr(
self.litellm_logging_obj, "_on_detached_stream_failure", None
)
if on_failure is None:
return
try:
with anyio.CancelScope(shield=True):
await on_failure(failure)
except Exception as hook_failure: # noqa: BLE001 # a failing proxy hook must not hide the error event from the client
verbose_logger.warning(
"AnthropicResponsesStreamWrapper: stream failure hook raised: %s(%s)",
type(hook_failure).__name__,
hook_failure,
)
def __aiter__(self) -> "AnthropicResponsesStreamWrapper":
return self
@ -450,6 +488,7 @@ class AnthropicResponsesStreamWrapper:
if self._chunk_queue:
return self._chunk_queue.popleft()
if self._stream_failed:
await self._report_unreported_failure()
raise StopAsyncIteration
if not self._sent_message_start:
@ -478,7 +517,7 @@ class AnthropicResponsesStreamWrapper:
verbose_logger.exception(
"AnthropicResponsesStreamWrapper: upstream Responses stream for %s failed", self.model
)
self._fail_stream(*_failure_status_and_message(e))
self._fail_stream(*_failure_status_and_message(e), failure=_original_failure(e))
if not self._chunk_queue and not self._sent_message_stop and not self._stream_failed:
verbose_logger.error(
@ -498,8 +537,14 @@ class AnthropicResponsesStreamWrapper:
async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
async for chunk in self:
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
try:
async for chunk in self:
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
finally:
from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt
if not closing_superseded_attempt():
await self._report_unreported_failure()
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
return partial_responses_incomplete_event_on_disconnect(self.responses_stream)

131
litellm/llms/azure/common_utils.py
View file

@ -5,7 +5,6 @@ import os
import re
from collections.abc import Awaitable, Callable, Mapping
from contextlib import closing
from functools import lru_cache
from types import MappingProxyType
from typing import Final, Literal, NamedTuple, cast
@ -18,15 +17,18 @@ import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.single_flight import SingleFlight, single_flight_lru_cache
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient
from litellm.secret_managers.get_azure_ad_token_provider import (
get_azure_ad_token_provider,
single_flight_bearer_token_provider,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.secret_managers.get_azure_ad_token_provider import AzureCredentialType
from litellm.utils import add_path_to_api_base
azure_ad_cache: Final = DualCache()
@ -94,7 +96,7 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
return {**llm_response_headers, **openai_headers}
@lru_cache(maxsize=128)
@single_flight_lru_cache
def _cached_entra_id_token_provider(
tenant_id: str,
client_id: str,
@ -107,33 +109,38 @@ def _cached_entra_id_token_provider(
to expiry, so reusing the provider keeps one AAD round trip per token lifetime instead of one
per request.
"""
from azure.identity import ClientSecretCredential, get_bearer_token_provider
from azure.identity import ClientSecretCredential
return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
return single_flight_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
@lru_cache(maxsize=128)
@single_flight_lru_cache
def _cached_username_password_token_provider(
client_id: str,
azure_username: str,
azure_password: str,
scope: str,
) -> Callable[[], str]:
from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
from azure.identity import UsernamePasswordCredential
credential: Final = UsernamePasswordCredential(
client_id=client_id,
username=azure_username,
password=azure_password,
)
return get_bearer_token_provider(credential, scope)
return single_flight_bearer_token_provider(credential, scope)
@lru_cache(maxsize=128)
@single_flight_lru_cache
def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
return get_azure_ad_token_provider(azure_scope=scope)
@single_flight_lru_cache
def _cached_default_azure_credential_provider(scope: str) -> Callable[[], str]:
return get_azure_ad_token_provider(azure_scope=scope, azure_credential=AzureCredentialType.DefaultAzureCredential)
def cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
return _cached_azure_ad_token_refresh_provider(scope)
@ -240,6 +247,58 @@ def get_azure_ad_token_from_username_password(
return token_provider
@single_flight_lru_cache
def _oidc_token_exchange(cache_key: str) -> SingleFlight[str]:
return SingleFlight()
def _cached_oidc_token(cache_key: str) -> str | None:
cached: Final = azure_ad_cache.get_cache(cache_key)
return cached if isinstance(cached, str) else None
def _exchange_oidc_token(cache_key: str, token_url: str, client_id: str, scope: str, oidc_token: str) -> str:
cached_access_token: Final = _cached_oidc_token(cache_key)
if cached_access_token is not None:
return cached_access_token
with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
req_token: Final = client.post(
token_url,
data={
"client_id": client_id,
"grant_type": "client_credentials",
"scope": scope,
"client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
"client_assertion": oidc_token,
},
)
if req_token.status_code != 200:
raise AzureOpenAIError(
status_code=req_token.status_code,
message=req_token.text,
)
azure_ad_token_json: Final[_AzureAdTokenJson] = req_token.json()
azure_ad_token_access_token: Final = azure_ad_token_json.get("access_token", None)
azure_ad_token_expires_in: Final = azure_ad_token_json.get("expires_in", None)
if azure_ad_token_access_token is None:
raise AzureOpenAIError(status_code=422, message="Azure AD Token access_token not returned")
if azure_ad_token_expires_in is None:
raise AzureOpenAIError(status_code=422, message="Azure AD Token expires_in not returned")
azure_ad_cache.set_cache(
key=cache_key,
value=azure_ad_token_access_token,
ttl=azure_ad_token_expires_in,
)
return azure_ad_token_access_token
def get_azure_ad_token_from_oidc(
azure_ad_token: str,
azure_client_id: str | None = None,
@ -282,50 +341,25 @@ def get_azure_ad_token_from_oidc(
"azure_client_id": azure_client_id,
"azure_tenant_id": azure_tenant_id,
"azure_authority_host": azure_authority_host,
"scope": scope,
"oidc_token": oidc_token,
}
)
azure_ad_token_access_token = azure_ad_cache.get_cache(azure_ad_token_cache_key)
if azure_ad_token_access_token is not None:
return azure_ad_token_access_token
cached_access_token: Final = _cached_oidc_token(azure_ad_token_cache_key)
if cached_access_token is not None:
return cached_access_token
with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
req_token: Final = client.post(
f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
data={
"client_id": azure_client_id,
"grant_type": "client_credentials",
"scope": scope,
"client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
"client_assertion": oidc_token,
},
return _oidc_token_exchange(azure_ad_token_cache_key).run(
lambda: _exchange_oidc_token(
azure_ad_token_cache_key,
token_url=f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
client_id=azure_client_id,
scope=scope,
oidc_token=oidc_token,
)
if req_token.status_code != 200:
raise AzureOpenAIError(
status_code=req_token.status_code,
message=req_token.text,
)
azure_ad_token_json: Final[_AzureAdTokenJson] = req_token.json()
azure_ad_token_access_token = azure_ad_token_json.get("access_token", None)
azure_ad_token_expires_in: Final = azure_ad_token_json.get("expires_in", None)
if azure_ad_token_access_token is None:
raise AzureOpenAIError(status_code=422, message="Azure AD Token access_token not returned")
if azure_ad_token_expires_in is None:
raise AzureOpenAIError(status_code=422, message="Azure AD Token expires_in not returned")
azure_ad_cache.set_cache(
key=azure_ad_token_cache_key,
value=azure_ad_token_access_token,
ttl=azure_ad_token_expires_in,
)
return azure_ad_token_access_token
def select_azure_base_url_or_endpoint(azure_client_params: dict):
azure_endpoint: Final = azure_client_params.get("azure_endpoint", None)
@ -503,17 +537,10 @@ class BaseAzureLLM(BaseOpenAILLM):
Returns:
Token provider callable if DefaultAzureCredential is enabled and available, None otherwise
"""
from litellm.types.secret_managers.get_azure_ad_token_provider import (
AzureCredentialType,
)
verbose_logger.debug("Attempting to use DefaultAzureCredential for Azure Auth")
try:
azure_ad_token_provider: Final = get_azure_ad_token_provider(
azure_scope=scope,
azure_credential=AzureCredentialType.DefaultAzureCredential,
)
azure_ad_token_provider: Final = _cached_default_azure_credential_provider(scope)
verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential")
return azure_ad_token_provider
except Exception as e:

8
litellm/llms/base_llm/agents/transformation.py
View file

@ -15,6 +15,7 @@ from collections.abc import Mapping
import httpx
from litellm.types.agents import (
AgentCreateRequest,
AgentCreateResponse,
AgentDeleteResult,
AgentListResponse,
@ -50,10 +51,9 @@ class BaseAgentsAPIConfig(ABC):
@abstractmethod
def transform_create_request(
self,
name: str,
litellm_params: Mapping[str, object],
) -> dict[str, object]:
"""Map name + litellm_params to the provider's create-agent body."""
create_request: AgentCreateRequest,
) -> Mapping[str, object]:
"""Map the create request to the provider's create-agent body."""
@abstractmethod
def transform_create_response(

23
litellm/llms/base_llm/files/transformation.py
View file

@ -1,6 +1,7 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Union
from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from openai.types.file_deleted import FileDeleted
@ -38,6 +39,22 @@ else:
UserAPIKeyAuth = Any
def _routes_nowhere(_model_name: str) -> bool:
return False
@dataclass(frozen=True, slots=True)
class FileListModelScope:
model_mapping_keys: frozenset[str]
routes_stored_model_name: Callable[[str], bool] = _routes_nowhere
def holds(self, model_mapping_keys: Iterable[str], stored: bool) -> bool:
mapped_keys: Final = frozenset(model_mapping_keys)
return not self.model_mapping_keys.isdisjoint(mapped_keys) or (
stored and any(map(self.routes_stored_model_name, mapped_keys))
)
class BaseFileUploadStream(ABC):
"""Re-iterable request body that yields an upload's bytes lazily.
@ -274,6 +291,8 @@ class BaseFileEndpoints(ABC):
user_api_key_dict: UserAPIKeyAuth,
limit: int | None = None,
after: str | None = None,
model_scope: FileListModelScope | None = None,
before: str | None = None,
**data: dict,
) -> FileListPage:
pass

23
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
View file

@ -10,6 +10,7 @@ from typing import Any, Final
from pydantic import JsonValue
from typing_extensions import assert_never
import litellm
from litellm._logging import verbose_logger
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields
@ -53,6 +54,18 @@ class CountTokensFailure:
CountTokensOutcome = CountedTokens | CountTokensFailure
def _claude_count_translator(litellm_params: Mapping[str, object]) -> AnthropicConfig:
deployment_model: Final = litellm_params.get("model")
if not isinstance(deployment_model, str):
return AnthropicConfig()
provider_hint: Final = litellm_params.get("custom_llm_provider")
model, provider, _, _ = litellm.get_llm_provider(
model=deployment_model, custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None
)
chat_config: Final = litellm.ProviderConfigManager.get_provider_chat_config(model, LlmProviders(provider))
return chat_config if isinstance(chat_config, AnthropicConfig) else AnthropicConfig()
def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model: str) -> bool:
return (
isinstance(outcome, CountTokensFailure)
@ -138,16 +151,22 @@ class BedrockTokenCounter(BaseTokenCounter):
counted_model: Final = model_to_use.removeprefix("bedrock_mantle/")
resolved_model: Final = get_bedrock_base_model(counted_model)
primary_tokenizer: Final = MANTLE_TOKENIZER_TYPE if use_mantle else RUNTIME_TOKENIZER_TYPE
litellm_params: Final = (deployment or {}).get("litellm_params", {})
translated: Final = (
await anthropic_count_fields(
counted_model, messages, system, tools, messages_format, AnthropicConfig(), chat_params
counted_model,
messages,
system,
tools,
messages_format,
_claude_count_translator(litellm_params),
chat_params,
)
if is_mantle_claude_model(resolved_model)
else None
)
if isinstance(translated, ChatCountTranslationError):
return translated.response(request_model, model_to_use, primary_tokenizer)
litellm_params: Final = (deployment or {}).get("litellm_params", {})
runtime_request: Final = (
_request_data(model_to_use, messages, tools, system)
if translated is None

40
litellm/llms/gemini/agents/transformation.py
View file

@ -20,16 +20,13 @@ from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
from litellm.types.agents import (
AgentCreateRequest,
AgentCreateResponse,
AgentDeleteResult,
AgentListResponse,
AgentVersionsResponse,
)
# Keys inside litellm_params that should be forwarded to the Gemini
# create-agent body verbatim.
_GEMINI_AGENT_BODY_KEYS: Final = ("base_agent", "instructions", "base_environment")
class _JsonResponse(Protocol):
def json(self) -> object: ...
@ -39,30 +36,6 @@ def _json_body(response: _JsonResponse) -> object:
return response.json()
# LiteLLM-internal keys that must never be forwarded to Gemini.
_LITELLM_INTERNAL_KEYS: Final = frozenset(
{
"custom_llm_provider",
"api_key",
"api_base",
"make_public",
"cost_per_query",
"input_cost_per_token",
"output_cost_per_token",
"require_trace_id_on_calls_to_agent",
"require_trace_id_on_calls_by_agent",
"max_iterations",
"max_budget_per_session",
"guardrails",
"is_public",
"agent_name",
"agent_id",
"agent_card_params",
"provider_agent_response",
}
)
class GeminiAgentsConfig(BaseAgentsAPIConfig):
"""
Configuration for the Google AI Studio (Gemini) native Agents API.
@ -143,14 +116,9 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig):
def transform_create_request(
self,
name: str,
litellm_params: Mapping[str, object],
) -> dict[str, object]:
body: Final[dict[str, object]] = {"name": name}
for key in _GEMINI_AGENT_BODY_KEYS:
value = litellm_params.get(key)
if value is not None:
body[key] = value
create_request: AgentCreateRequest,
) -> Mapping[str, object]:
body: Final[Mapping[str, object]] = create_request.model_dump(mode="json", exclude_none=True)
verbose_logger.debug("GeminiAgentsConfig create body: %s", body)
return body

20
litellm/llms/gemini/interactions/transformation.py
View file

@ -15,7 +15,7 @@ import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
@ -41,6 +41,14 @@ else:
_JsonObject: TypeAlias = dict[str, object]
def _has_agent_config_type(agent_config: object) -> bool:
match agent_config:
case {"type": str()}:
return True
case _:
return False
class _InteractionPayload(TypedDict, total=False):
"""JSON body of an Interactions API interaction, keyed as ``InteractionsAPIResponse`` fields."""
@ -105,6 +113,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
return [
"model",
"agent",
"agent_config",
"input",
"tools",
"system_instruction",
@ -188,6 +197,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
request_body["input"] = input
optional_keys: Final = [
"agent_config",
"tools",
"system_instruction",
"stream",
@ -197,8 +207,14 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
"response_modalities",
"previous_interaction_id",
]
dropped_keys: Final = (
frozenset({"agent_config"})
if drops_unsupported_params(litellm_params.get("drop_params"))
and (model or not _has_agent_config_type(optional_params.get("agent_config")))
else frozenset()
)
for key in optional_keys:
if optional_params.get(key) is not None:
if key not in dropped_keys and optional_params.get(key) is not None:
request_body[key] = optional_params[key]
# response_mime_type is removed — fold it into response_format.

17
litellm/llms/ollama/decisions/transformation.py Normal file
View file

@ -0,0 +1,17 @@
from typing import Final
from litellm.constants import PLACEHOLDER_API_KEY
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class OllamaDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("OLLAMA_API_KEY",)
api_base_env = ("OLLAMA_API_BASE",)
api_key_required = False
def get_default_api_base(self) -> str | None:
return "http://localhost:11434"
def resolve_api_key(self, api_key: str | None) -> str | None:
resolved: Final = super().resolve_api_key(api_key)
return None if resolved == PLACEHOLDER_API_KEY else resolved

10
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -76,6 +76,7 @@ else:
_NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({})
_OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_DECODED_JSON: Final = TypeAdapter(object)
@ -958,8 +959,13 @@ class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
status_code=error_status_code,
message=error_message,
)
choices = chunk.get("choices", [])
choices = self._map_reasoning_to_reasoning_content(choices)
raw_choices: Final = chunk.get("choices") or []
if any(choice.get("finish_reason") == "abort" for choice in _CHOICE_MAPPINGS.validate_python(raw_choices)):
raise OpenAIError(
status_code=500,
message="Upstream generation aborted (finish_reason=abort)",
)
choices: Final = self._map_reasoning_to_reasoning_content(raw_choices)
fingerprint: Final = _OBJECT_MAPPING.validate_python(chunk).get("system_fingerprint")
kwargs: Final[dict[str, Any]] = {

6
litellm/llms/openai_like/providers.json
View file

@ -58,6 +58,12 @@
"max_completion_tokens": "max_tokens"
}
},
"zerogpu": {
"base_url": "https://api.zerogpu.ai/v1",
"api_key_env": "ZEROGPU_API_KEY",
"api_base_env": "ZEROGPU_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
},
"chutes": {
"base_url": "https://llm.chutes.ai/v1/",
"api_key_env": "CHUTES_API_KEY",

25
litellm/main.py
View file

@ -6127,7 +6127,18 @@ def completion(
optional_params, rs_val = strip_reasoning_summary_aliases_from_optional_params(optional_params)
if isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
if custom_llm_provider == "openai" and isinstance(resolved_reasoning_effort, dict):
from litellm.responses.utils import openai_bridge_reasoning
optional_params["reasoning_effort"] = openai_bridge_reasoning(
requested=resolved_reasoning_effort,
effort_kept="reasoning_effort" in optional_params,
summary_alias=rs_val,
model=model,
drop_params=drop_unsupported_reasoning,
additional_drop_params=routing_kwargs.get("additional_drop_params"),
)
elif isinstance(resolved_reasoning_effort, dict) and "summary" in resolved_reasoning_effort:
optional_params["reasoning_effort"] = resolved_reasoning_effort
elif rs_val is not None:
eff: Final = optional_params.get("reasoning_effort", resolved_reasoning_effort)
@ -8394,7 +8405,7 @@ def transcription(
custom_llm_provider=custom_llm_provider,
)
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] | None = None
response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]
provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model,
@ -8556,10 +8567,16 @@ def transcription(
provider_config=provider_config,
shared_session=shared_session,
)
else:
raise litellm.UnsupportedEndpointError(
message=f"{custom_llm_provider}/{model} does not support audio transcription.",
model=model,
llm_provider=custom_llm_provider,
)
# Store duration in _hidden_params for cost calculation without
# exposing it in the response body (see sync path comment above).
if response is not None and not isinstance(response, Coroutine):
if not isinstance(response, Coroutine):
existing_duration: Final = getattr(response, "duration", None)
if existing_duration is None:
calculated_duration: Final = calculate_request_duration(file)
@ -8569,8 +8586,6 @@ def transcription(
)
response_hidden_params["audio_transcription_duration"] = calculated_duration
if response is None:
raise ValueError("Unmapped provider passed in. Unable to get the response.")
return response

1527
litellm/model_prices_and_context_window_backup.json
View file

File diff suppressed because it is too large Load diff

28
litellm/provider_endpoints_support_backup.json
View file

@ -1225,7 +1225,7 @@
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_generations": true,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
@ -4437,6 +4437,32 @@
"generateContent": true
}
},
"zerogpu": {
"display_name": "ZeroGPU (`zerogpu`)",
"url": "https://docs.litellm.ai/docs/providers/zerogpu",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"ragflow": {
"display_name": "RAGFlow (`ragflow`)",
"url": "https://docs.litellm.ai/docs/providers/ragflow",

2
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -15015,7 +15015,7 @@
]
},
"post": {
"description": "Create a named custom agent on the Gemini side.\n\nExample:\n```bash\ncurl -X POST \"http://localhost:4000/v1beta/agents\" \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"my-custom-slides-agent\",\n \"base_agent\": \"waverunner\",\n \"instructions\": \"You are a helpful assistant that creates slides.\",\n \"base_environment\": {\n \"type\": \"remote\",\n \"sources\": [\n {\"type\": \"gcs\", \"source\": \"gs://eap-templates/slides-skill\",\n \"target\": \"/.agents/skills/slides-skill\"}\n ]\n }\n }'\n```",
"description": "Create a named custom agent on the Gemini side.\n\nExample:\n```bash\ncurl -X POST \"http://localhost:4000/v1beta/agents\" \\\n -H \"Authorization: Bearer sk-...\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"id\": \"my-custom-slides-agent\",\n \"base_agent\": \"antigravity-preview-09-2026\",\n \"system_instruction\": \"You are a helpful assistant that creates slides.\",\n \"agent_config\": {\"type\": \"antigravity\", \"model\": \"gemini-3.8-flash\"},\n \"base_environment\": {\n \"type\": \"remote\",\n \"sources\": [\n {\"type\": \"gcs\", \"source\": \"gs://eap-templates/slides-skill\",\n \"target\": \"/.agents/skills/slides-skill\"}\n ]\n }\n }'\n```",
"operationId": "create_gemini_agent_v1beta_agents_post",
"responses": {
"200": {

4
litellm/proxy/client/README.md
View file

@ -342,7 +342,7 @@ The CLI provides these authentication commands:
- **`lite login --pkce`** - Sign in through the system browser with OAuth authorization code + PKCE; the key renews itself with a refresh token
- **`lite logout`** - Clear stored authentication token (and revoke a `--pkce` refresh token on the proxy)
- **`lite whoami`** - Show current authentication status
- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); renews a `--pkce` key first and fails once a classic token has expired
- **`lite auth print-token`** - Print the cached token (used as Claude Code's `apiKeyHelper`); renews a `--pkce` key first, and fails once a classic token is past 24 hours and the proxy rejects it
### Authentication Flow Steps
@ -382,7 +382,7 @@ Keychain storage needs the `keyring` package, which ships with `pip install 'lit
`lite logout` clears both stores. If the keychain is locked at that moment it says so, and re-running it once the keychain is unlocked finishes the job.
The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while fresh and fails once it expires -- there is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. A credential from `lite login --pkce` is the exception: it carries a refresh token, so the CLI renews the key shortly before it expires and `lite logout` revokes the refresh token on the proxy (see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
The stored credential is a short-lived, per-session agent token, not a managed virtual key. It is scoped to the user and team you logged in as and inherits their models and budgets; spend is tracked against the shared team and user budgets rather than a separate per-session cap, so multiple logins or several concurrent agents all draw down the same allowance. It is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); re-run `lite login` to refresh it and pick up your latest team and user settings. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it, and once it is past 24 hours (or the client's `LITELLM_CLI_JWT_EXPIRATION_HOURS`) asks the proxy first and fails if the proxy rejects it. There is no silent renewal. It is accepted on a default deployment without `EXPERIMENTAL_UI_LOGIN`, does not appear in the Keys UI, and cannot be rotated or revoked mid-session. A credential from `lite login --pkce` is the exception: it carries a refresh token, so the CLI renews the key shortly before it expires and `lite logout` revokes the refresh token on the proxy (see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. For a long-lived, rotatable, Keys-UI-visible credential, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY`.
### Usage

2
litellm/proxy/client/cli/README.md
View file

@ -506,7 +506,7 @@ To pin the model, pass the agent's own model flag (for example `lite claude --mo
The token minted by `lite login` is a short-lived, per-session agent credential, not a managed virtual key. It is scoped to the user and team you authenticated as, inherits that user's and team's models and budgets, and is enforced on the proxy exactly like a virtual key on the same team (guardrails, routing, logging, spend). Spend is tracked against the shared team and user budgets, so running several agents (or logging in more than once) does not hand each session its own separate budget; they all draw down the same team/user allowance. There is no separate per-session cap, so sustained agent use is not capped at a small chat-session limit.
The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while it's still fresh and fails once it expires -- there is no silent renewal, so a long-running session needs a fresh `lite login` once a day. `lite claude`, `lite codex`, `lite opencode`, and `lite pi` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. `lite login --pkce` is the exception to the daily re-login: it signs in through your system browser with OAuth authorization code and PKCE and stores a refresh token next to the key, so every `lite` command and `lite auth print-token` renew the key on their own shortly before it expires, `lite whoami` shows when the current key expires, and `lite logout` revokes the refresh token on the proxy (it needs a proxy that serves `/.well-known/litellm-cli-auth`; see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). When a renewal is refused, for example after a `lite logout` run from another copy of the credential, the command prints why on stderr and, once the key has run out, tells you to run `lite login --pkce` again. Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it without asking the proxy for its first 24 hours, or `LITELLM_CLI_JWT_EXPIRATION_HOURS` if you set it on the client. After that, `print-token` and `lite whoami` ask the proxy whether it still accepts the token, and `print-token` fails once the proxy rejects it. There is no silent renewal, so a long-running session needs a fresh `lite login` each time the proxy's token lifetime runs out. If the proxy's tokens last less than 24 hours, set the same `LITELLM_CLI_JWT_EXPIRATION_HOURS` on the client so `print-token` stops printing the token once it runs out. `lite up` still starts a new login once the token is older than that same client window. It copies the token into Claude Code's settings as a static key that it never renews, so a fresh login gives Claude Code the proxy's full token lifetime instead of whatever an old token has left. `lite claude`, `lite codex`, `lite opencode`, and `lite pi` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. `lite login --pkce` is the exception to the re-login: it signs in through your system browser with OAuth authorization code and PKCE and stores a refresh token next to the key, so every `lite` command and `lite auth print-token` renew the key on their own shortly before it expires, `lite whoami` shows when the current key expires, and `lite logout` revokes the refresh token on the proxy (it needs a proxy that serves `/.well-known/litellm-cli-auth`; see [Browser sign-in with PKCE](https://docs.litellm.ai/docs/proxy/cli_sso#browser-sign-in-with-pkce)). When a renewal is refused, for example after a `lite logout` run from another copy of the credential, the command prints why on stderr and, once the key has run out, tells you to run `lite login --pkce` again. Only the holder can end a `--pkce` session early, with `lite logout`; an admin has no button for it, but every renewal re-reads the user on the proxy, so deactivating the user or removing them from the team makes the next renewal fail and the key runs out within `LITELLM_CLI_JWT_EXPIRATION_HOURS`. On a proxy with more than one worker or replica, configure Redis (`litellm_settings.cache` with Redis `cache_params`, or `general_settings.coordination_redis`) so a refresh token stays single-use and `lite logout` holds on every worker; without Redis each worker keeps its own record. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
### Route Every Claude Code Session Through the Proxy

44
litellm/proxy/client/cli/commands/auth.py
View file

@ -13,7 +13,6 @@ from rich.console import Console
from rich.table import Table
from typing_extensions import NotRequired, ReadOnly, TypedDict, assert_never
from litellm.constants import CLI_JWT_EXPIRATION_HOURS
from litellm.litellm_core_utils.cli_keyring import (
DISABLE_KEYRING_ENV_VAR,
SYSTEM_KEYRING,
@ -143,6 +142,9 @@ _TeamMapping: Final = TypeVar("_TeamMapping", bound=Mapping[str, object])
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_PROXY_KEY_CHECK_TIMEOUT_SECONDS: Final = 3
_PROXY_REJECTED_KEY_STATUSES: Final = frozenset({401, 403})
KEYRING_INSTALL_HINT: Final = "pip install 'litellm[cli]'"
KEYRING_ENABLE_HINT: Final = "keyring --enable (or unset PYTHON_KEYRING_BACKEND)"
@ -1033,10 +1035,12 @@ def print_token(ctx: click.Context):
Designed to be used as Claude Code's `apiKeyHelper`
(https://docs.claude.com/en/docs/claude-code/settings): stdout must
contain only the token, so all diagnostics go to stderr. The token
expires after `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h); a
`lite login --pkce` token renews itself here first, and once a token
has expired for good, run the same `lite login` command again.
contain only the token, so all diagnostics go to stderr. A token younger
than `LITELLM_CLI_JWT_EXPIRATION_HOURS` (default 24h) is printed without
asking the proxy. An older one is checked against the proxy and refused
only if the proxy rejects it. A `lite login --pkce`
token renews itself here first, and once a token has expired for good,
run the same `lite login` command again.
"""
vault: Final = context_secret_vault(ctx)
token_data: Final = load_token(vault=vault)
@ -1055,7 +1059,7 @@ def print_token(ctx: click.Context):
sys.exit(1)
renews: Final = "refresh_token" in token_data
if not is_cli_token_fresh(token_data) and not renews:
if not renews and token_expired(token_data):
click.echo("Token expired. Run 'lite login' again.", err=True)
sys.exit(1)
@ -1111,8 +1115,32 @@ def whoami(ctx: click.Context):
expires_at: Final = token_data.get("expires_at")
if isinstance(expires_at, (int, float)):
click.echo(_key_expiry_line(expires_at, renews="refresh_token" in token_data))
elif age_hours > CLI_JWT_EXPIRATION_HOURS:
click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
elif token_expired(token_data):
click.echo("Token expired. Run 'lite login' again.")
def token_expired(token_data: Mapping[str, object]) -> bool:
if is_cli_token_fresh(token_data):
return False
if isinstance(token_data.get("expires_at"), (int, float)):
return True
return _proxy_rejects_key(token_data)
def _proxy_rejects_key(token_data: Mapping[str, object]) -> bool:
key: Final = token_data.get("key")
base_url: Final = token_data.get("base_url")
if not isinstance(key, str) or not isinstance(base_url, str) or not base_url:
return True
try:
response: Final = requests.get(
f"{base_url}/v1/models",
headers={"Authorization": f"Bearer {key}"},
timeout=_PROXY_KEY_CHECK_TIMEOUT_SECONDS,
)
except requests.RequestException:
return False
return response.status_code in _PROXY_REJECTED_KEY_STATUSES
def _key_expiry_line(expires_at: float, renews: bool) -> str:

89
litellm/proxy/common_request_processing.py
View file

@ -4,6 +4,7 @@ import json
import logging
import math
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Coroutine, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from functools import lru_cache
from itertools import chain
@ -136,7 +137,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
_check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
check_and_merge_model_level_guardrails,
)
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
from litellm.router import Router
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
@ -2006,6 +2007,49 @@ def _content_text(fields: Mapping[str, object]) -> str:
return str(fields.get("content", ""))
async def _no_stream_failure_to_release() -> None:
return None
@dataclass(slots=True)
class _ClientStreamFailureReport:
logging_obj: LiteLLMLoggingObj
report: Callable[[Exception], Awaitable[None]]
held: Exception | None = None
@classmethod
def hold_while_streaming(cls, logging_obj: LiteLLMLoggingObj | None) -> Callable[[], Awaitable[None]]:
if logging_obj is None or logging_obj.on_detached_stream_failure is None:
return _no_stream_failure_to_release
return cls(logging_obj, logging_obj.on_detached_stream_failure)._start_holding()
def _start_holding(self) -> Callable[[], Awaitable[None]]:
self.logging_obj.on_detached_stream_failure = self.hold
return self.release
async def hold(self, failure: Exception) -> None:
if self.held is None:
self.held = failure
async def release(self) -> None:
self.logging_obj.on_detached_stream_failure = self.report
if self.held is None:
return
await asyncio.shield(
spawn_logging_task(
asyncio.get_running_loop(), self._report_held(self.held), task_name="stream failure report"
)
)
async def _report_held(self, failure: Exception) -> None:
try:
await self.report(failure)
except Exception as hook_failure: # noqa: BLE001 # a failing failure hook must not skip the stream cleanup
verbose_proxy_logger.warning(
"Stream failure hook raised: %s(%s)", type(hook_failure).__name__, sanitize_for_log(hook_failure)
)
class ProxyBaseLLMRequestProcessing:
def __init__(self, data: dict, served_model: str | None = None):
self.data = data
@ -3498,6 +3542,8 @@ class ProxyBaseLLMRequestProcessing:
request_data: Final = self.data
async def _on_detached_stream_failure(exc: Exception) -> None:
if not logging_obj.claim_stream_failure_report():
return
try:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
@ -4448,11 +4494,15 @@ class ProxyBaseLLMRequestProcessing:
client_disconnected = False
delivered_chunk = False
recent_tail = SSE_STREAM_START_TAIL # rebind-ok: rolling window over the yielded bytes
stream_logging_obj: Final[object] = request_data.get("litellm_logging_obj") # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # legacy request data is narrowed to Logging below
guarded_stream: Final[AsyncGenerator[object, None]] = proxy_logging_obj.async_post_call_streaming_iterator_hook(
user_api_key_dict=user_api_key_dict,
response=response,
request_data=request_data,
)
release_stream_failure: Final = _ClientStreamFailureReport.hold_while_streaming(
stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None
)
try:
str_so_far = ""
async for chunk in guarded_stream:
@ -4523,10 +4573,18 @@ class ProxyBaseLLMRequestProcessing:
verbose_proxy_logger.exception(
"litellm.proxy.proxy_server.async_data_generator(): Exception occured - %s", e
)
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=request_data,
already_reported: Final = (
isinstance(stream_logging_obj, LiteLLMLoggingObj)
and not stream_logging_obj.claim_stream_failure_report()
)
transformed_exception: Final = (
None
if already_reported
else await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=request_data,
)
)
if transformed_exception is not None:
e = transformed_exception
@ -4549,15 +4607,18 @@ class ProxyBaseLLMRequestProcessing:
seal: Final = "" if seal_open_frame is None else seal_open_frame(recent_tail)
yield seal + error_frame if seal else error_frame
finally:
await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
request=request,
request_data=request_data,
response=response,
stream_completed=stream_completed,
client_disconnected=client_disconnected,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
)
try:
await release_stream_failure()
finally:
await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup(
request=request,
request_data=request_data,
response=response,
stream_completed=stream_completed,
client_disconnected=client_disconnected,
user_api_key_dict=user_api_key_dict,
proxy_logging_obj=proxy_logging_obj,
)
@staticmethod
def async_sse_data_generator(

15
litellm/proxy/common_utils/token_count_settings.py
View file

@ -13,6 +13,7 @@ from litellm.types.token_count import TokenCountProvider, TokenCountSourceFields
from litellm.types.utils import TokenCountResponse
CANNOT_COUNT_MODEL_STATUSES: Final = frozenset({400, 404})
REGION_DENIED_STATUSES: Final = frozenset({401, 403})
TOKENIZER_LOAD_WAIT_SECONDS: Final = 5.0
TOKENIZER_LOAD_REUSE_SECONDS: Final = 60.0
TOKENIZER_LOAD_MAX_RUNNING: Final = 4
@ -47,6 +48,7 @@ class TokenCountModel:
model: str
region: str | None
litellm_provider: TokenCountProvider | None = None
configured: bool = False
@dataclass(frozen=True, slots=True)
@ -66,7 +68,13 @@ class ProviderCountFailure:
def cannot_count_model(self) -> bool:
return not self.rejected_before_provider_call and self.status_code in CANNOT_COUNT_MODEL_STATUSES
@property
def denied_in_region(self) -> bool:
return not self.rejected_before_provider_call and self.status_code in REGION_DENIED_STATUSES
def with_hint(self, hint: str) -> "ProviderCountFailure":
if self.rejected_before_provider_call:
return self
return replace(self, message=_with_hint(self.message, hint))
def to_proxy_exception(self) -> ProxyException:
@ -111,7 +119,7 @@ def _count_provider(value: object) -> TokenCountProvider | None:
return None
def _count_model(value: Mapping[str, object]) -> TokenCountModel | None:
def _count_model(value: Mapping[str, object], configured: bool) -> TokenCountModel | None:
model: Final = _non_blank(value.get("model"))
return (
None
@ -120,6 +128,7 @@ def _count_model(value: Mapping[str, object]) -> TokenCountModel | None:
model=model,
region=_non_blank(value.get("region")),
litellm_provider=_count_provider(value.get("litellm_provider")),
configured=configured,
)
)
@ -146,12 +155,12 @@ def _resolved_count_model(
) -> TokenCountModel | None:
configured_mapping: Final = as_str_mapping(configured)
if configured_mapping is not None:
return _count_model(configured_mapping)
return _count_model(configured_mapping, configured=True)
default_mapping: Final = as_str_mapping(default)
return (
None
if default_mapping is None
else _within_deployment_geography(_count_model(default_mapping), deployment_region)
else _within_deployment_geography(_count_model(default_mapping, configured=False), deployment_region)
)

7
litellm/proxy/google_endpoints/agents_endpoints.py
View file

@ -160,9 +160,10 @@ async def create_gemini_agent(
-H "Authorization: Bearer sk-..." \\
-H "Content-Type: application/json" \\
-d '{
"name": "my-custom-slides-agent",
"base_agent": "waverunner",
"instructions": "You are a helpful assistant that creates slides.",
"id": "my-custom-slides-agent",
"base_agent": "antigravity-preview-09-2026",
"system_instruction": "You are a helpful assistant that creates slides.",
"agent_config": {"type": "antigravity", "model": "gemini-3.8-flash"},
"base_environment": {
"type": "remote",
"sources": [

35
litellm/proxy/guardrails/_content_utils.py
View file

@ -305,6 +305,11 @@ def iter_message_text(data: Mapping[str, object]) -> Iterator[str]:
yield from _iter_text_parts_in_content(message.get("content"))
def _is_tool_message(message: object) -> bool:
fields: Final = _text_object(message)
return fields is not None and str(fields.get("role") or "").lower() == "tool"
def _call_item(item: object) -> tuple[str, object] | None:
"""Return ``(field, value)`` for a Responses tool-call input item, else None."""
fields: Final = _text_object(item)
@ -316,11 +321,14 @@ def walk_user_text(
data: dict[str, object],
visit: Callable[[str], str],
visit_tool: Callable[[JsonValue], JsonValue] | None = None,
*,
skip_tool_message: bool = False,
) -> int:
"""Rewrite every text fragment in place via ``visit``.
Mutates ``data["messages"]`` and ``data["input"]`` (including tool payloads), ``data["system"]`` and
``data["instructions"]``. Returns the number
``data["instructions"]``. ``skip_tool_message`` leaves ``role: tool`` messages and ``tool_result``
blocks as sent. Returns the number
of fragments visited so callers can short-circuit when nothing was
inspected.
"""
@ -353,6 +361,8 @@ def walk_user_text(
def _rewrite_part(part: Mapping[str, object], native: bool) -> Mapping[str, object]:
nonlocal visited
if skip_tool_message and part.get("type") == "tool_result":
return part
text: Final = _part_text(part)
rewritten: Final = {**part, **({"text": visit(text)} if text is not None else {})}
if text is not None:
@ -384,18 +394,17 @@ def walk_user_text(
return content
messages: Final = data.get("messages")
if isinstance(messages, list):
for message in messages:
if not isinstance(message, dict):
continue
if "content" in message:
message["content"] = _rewrite_content(message["content"])
calls = message.get("tool_calls")
if isinstance(calls, list):
for call in calls:
function = call.get("function") if isinstance(call, dict) else None
if isinstance(function, dict) and "arguments" in function:
function["arguments"] = _rewrite_arguments(function["arguments"])
for message in _object_list(messages) or ():
if (skip_tool_message and _is_tool_message(message)) or not isinstance(message, dict):
continue
if "content" in message:
message["content"] = _rewrite_content(message["content"])
calls = message.get("tool_calls")
if isinstance(calls, list):
for call in calls:
function = call.get("function") if isinstance(call, dict) else None
if isinstance(function, dict) and "arguments" in function:
function["arguments"] = _rewrite_arguments(function["arguments"])
if "system" in data:
data["system"] = _rewrite_content(data["system"]) # rebind-ok: preserve the native hook's in-place contract

91
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -45,11 +45,14 @@ from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.cost_calculator import get_usage_object
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
split_media_for_tpm,
)
from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
from litellm.llms.azure.ptu_capacity import normalized_tokens
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
@ -98,6 +101,7 @@ if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
from litellm.router import Router
from litellm.types.agents import AgentResponse
from litellm.types.caching import RedisPipelineIncrementOperation
@ -135,6 +139,18 @@ class RateLimitedModel:
return requested_limit if requested_limit is not None else limits.get(self.group)
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
if router is None:
return (model,)
return tuple(dict.fromkeys(router.resolved_litellm_models(model, team_id, use_base_model=True))) or (model,)
def _resolve_media_models_via_proxy_router(model: str, team_id: str | None) -> tuple[str, ...]:
from litellm.proxy.proxy_server import llm_router
return _resolve_media_models(llm_router, model, team_id)
def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
from litellm.proxy.proxy_server import llm_router
@ -862,6 +878,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
time_provider: Callable[[], datetime] | None = None,
tag_rate_limit_resolver: TagRateLimitResolver = resolve_tag_rate_limits_from_db,
model_group_resolver: Callable[[str], str | None] = _resolve_model_group_alias_via_proxy_router,
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] = _resolve_media_models_via_proxy_router,
ptu_team_ceiling_resolver: Callable[
[str, str], PTUTeamCeiling | None
] = _resolve_ptu_team_ceiling_via_proxy_router,
@ -872,6 +889,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._time_provider = time_provider or datetime.now
self._tag_rate_limit_resolver = tag_rate_limit_resolver
self._model_group_resolver = model_group_resolver
self._media_model_resolver = media_model_resolver
self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
self._fail_closed_resolver = fail_closed_resolver
self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
@ -1133,11 +1151,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
def _estimate_tokens_for_request(
self,
data: dict,
data: object,
model: str | None = None,
min_configured_tpm_limit: int | None = None,
call_type: str | None = None,
configured_output_tokens: int | None = None,
team_id: str | None = None,
) -> int:
"""
Estimate total tokens this request will consume so we can reserve them
@ -1164,6 +1183,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
team_id=team_id,
media_model_resolver=self._media_model_resolver,
)
total_estimated: Final = estimated_input_tokens + max_tokens_estimate
@ -1184,6 +1205,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
call_type: str | None,
configured_output_tokens: int | None,
raw_estimate: int,
team_id: str | None = None,
) -> int:
"""The team PTU ceiling counts Azure normalized tokens, so its reservation weighs the
output budget the way the ceiling does instead of the raw sum the other scopes reserve."""
@ -1194,6 +1216,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
team_id=team_id,
media_model_resolver=self._media_model_resolver,
)
normalized: Final = normalized_tokens(
ceiling, prompt_tokens=estimated_input_tokens, completion_tokens=max_tokens_estimate
@ -1206,6 +1230,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
min_configured_tpm_limit: int | None = None,
call_type: str | None = None,
configured_output_tokens: int | None = None,
include_media: bool = True,
team_id: str | None = None,
media_model_resolver: Callable[[str, str | None], tuple[str, ...]] | None = None,
) -> tuple[int, int]:
"""
Estimate input tokens and output (max_tokens) budget separately, so
@ -1229,10 +1256,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
floor entirely, so the reservation reflects what this tenant's model
actually emits rather than one constant shared by every tenant.
"""
if not isinstance(data, dict):
request_data: Final = as_str_mapping(data)
if not isinstance(data, dict) or request_data is None:
return 0, 0
request_model: Final = request_data.get("model")
translated_data: Final = self._translate_google_genai_native_request(data, call_type)
estimable_data: Final = translated_data if translated_data is not None else data
estimable_data: Final = translated_data if translated_data is not None else request_data
selected_fields: Final[tuple[object | None, object | None, object | None]] = (
(None, None, estimable_data.get("input"))
if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES
@ -1249,10 +1278,27 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
)
messages, prompt, raw_input = selected_fields
input_text: Final = (
original_input: Final = (
billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input
)
media_source: Final = messages if messages is not None else original_input
projected, media = (
split_media_for_tpm(media_source)
if include_media and (isinstance(messages, list) or call_type in RESPONSES_API_CALL_TYPES)
else (original_input, ())
)
input_text: Final = projected if call_type in RESPONSES_API_CALL_TYPES else original_input
fallback_model: Final = request_model if isinstance(request_model, str) else None
resolved_models: Final = (
media_model_resolver(fallback_model or "", team_id)
if media and media_model_resolver is not None
else (fallback_model or "",)
)
media_tokens: Final = sum(
self._estimate_media_block_tokens(block, resolved_models, fallback_model) for block in media
)
total_chars: Final = (
len(get_str_from_messages(messages))
if isinstance(messages, list) and messages
@ -1267,7 +1313,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else 0
)
estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
estimated_input_tokens: Final = (
max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0
) + media_tokens
explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type)
generates_no_output: Final = (
@ -1283,7 +1331,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
max_tokens_estimate: Final = (
0
if generates_no_output
or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None)
or (explicit_max_tokens is None and estimated_input_tokens == 0 and configured_output_tokens is None)
else explicit_max_tokens
if explicit_max_tokens is not None
else configured_output_tokens
@ -1293,6 +1341,24 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return estimated_input_tokens, max_tokens_estimate * self.get_output_candidate_count(data, call_type)
@staticmethod
def _estimate_media_block_tokens(
block: Mapping[str, object], models: tuple[str, ...], fallback_model: str | None
) -> int:
if block.get("type") == "input_audio":
audio: Final = _callback_kwargs_mapping(block.get("input_audio"))
data: Final = audio.get("data")
return estimate_audio_tokens(data if isinstance(data, str) else None, models, fallback_model=fallback_model)
image: Final = _callback_kwargs_mapping(block.get("image_url"))
url: Final = image.get("url")
detail: Final = image.get("detail")
return estimate_image_tokens(
models,
url if isinstance(url, str) else "",
detail if isinstance(detail, str) else "auto",
fallback_model=fallback_model,
)
def _is_redis_cluster(self) -> bool:
"""
Check if the dual cache is using Redis cluster.
@ -3958,7 +4024,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
except Exception: # noqa: BLE001 # tokenizer failures degrade to the cheap estimate
if call_type in RERANK_API_CALL_TYPES and isinstance(selected_text, str):
return max(0, len(selected_text) // DEFAULT_CHARS_PER_TOKEN)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(data=data, call_type=call_type)
estimated_input_tokens, _ = self._estimate_input_and_output_tokens(
data=data, call_type=call_type, include_media=False
)
return estimated_input_tokens + audio_token_estimate
async def _reserve_project_io_tokens_or_raise(
@ -4003,6 +4071,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
data=data,
min_configured_tpm_limit=min_configured_otpm_limit,
call_type=call_type,
include_media=False,
)
raw_estimated_input_tokens: Final = await offload_token_count(self._estimate_precise_input_tokens)(
data=data, model=requested_model, call_type=call_type
@ -4360,12 +4429,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# Post-call reconciliation refunds the over-reservation
# delta when actual usage comes in below the floor.
estimated_tokens: Final = max(
self._estimate_tokens_for_request(
await offload_token_count(self._estimate_tokens_for_request)(
data=data,
model=requested_model,
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
team_id=user_api_key_dict.team_id,
),
1,
)
@ -4383,13 +4453,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
capped_request: Final = _REQUEST_RATE_LIMIT_DATA.validate_python(data)
ptu_estimated_tokens: Final = self._estimate_ptu_tokens_for_request(
ptu_estimated_tokens: Final = await offload_token_count(self._estimate_ptu_tokens_for_request)(
ceiling=stash.ptu_ceiling,
data=capped_request,
min_configured_tpm_limit=min_configured_tpm_limit,
call_type=call_type,
configured_output_tokens=configured_output_tokens,
raw_estimate=estimated_tokens,
team_id=user_api_key_dict.team_id,
)
if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit:
if not reuses_request_slot:

255
litellm/proxy/openai_files_endpoints/files_endpoints.py
View file

@ -6,9 +6,12 @@
######################################################################
import asyncio
import functools
import itertools
import traceback
from collections.abc import Mapping, Sequence
from typing import Any, BinaryIO, Final, TypedDict, cast, get_args
from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from typing import Any, BinaryIO, Final, NoReturn, TypedDict, cast, get_args
import httpx
from fastapi import (
@ -28,12 +31,13 @@ from typing_extensions import ReadOnly
import litellm
from litellm import CreateFileRequest, get_secret_str
from litellm._logging import verbose_proxy_logger
from litellm.constants import ANTHROPIC_DEFAULT_FILE_LIST_LIMIT
from litellm.litellm_core_utils.cloud_storage_security import (
is_managed_cloud_storage_uri,
)
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
from litellm.llms.base_llm.files.litellm_db_storage_backend import LITELLM_DB_STORAGE_BACKEND_NAME
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
from litellm.llms.base_llm.files.transformation import BaseFileEndpoints, FileListModelScope
from litellm.llms.base_llm.managed_resources.isolation import build_list_page
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@ -112,6 +116,9 @@ from litellm.types.llms.openai import (
OpenAIFileObject,
OpenAIFilesPurpose,
)
from litellm.types.proxy.auth.user_api_key_auth import user_api_key_has_admin_view
from litellm.types.router import Deployment
from litellm.types.utils import LlmProviders
router: Final = APIRouter()
@ -1673,6 +1680,185 @@ def _as_file_list_page(response: object) -> object:
return FileListPage.model_validate(page)
async def _list_owned_managed_files(
managed_files_obj: BaseFileEndpoints,
user_api_key_dict: UserAPIKeyAuth,
purpose: str | None,
limit: int | None,
after: str | None,
before: str | None,
model_scope: FileListModelScope | None,
) -> FileListPage:
return await managed_files_obj.afile_list(
purpose=purpose,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
user_api_key_dict=user_api_key_dict,
limit=limit,
after=after,
before=before,
model_scope=model_scope,
)
@dataclass(frozen=True, slots=True)
class _UnscopedFileList:
status_code: int
detail: str
@dataclass(frozen=True, slots=True)
class _ListedFiles:
model_scope: FileListModelScope | None
custom_llm_provider: str | None
def _raise_unscoped_file_list(failure: _UnscopedFileList) -> NoReturn:
raise HTTPException(status_code=failure.status_code, detail=failure.detail)
def _single_target_model_name(target_model_names: str) -> str | _UnscopedFileList:
target_model_names_list: Final = target_model_names.split(",")
if len(target_model_names_list) != 1:
return _UnscopedFileList(
status_code=400,
detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
)
return target_model_names_list[0]
async def _requested_custom_llm_provider(request: Request, provider: str | None) -> str | None:
return (
provider
or get_custom_llm_provider_from_request_headers(request=request)
or get_custom_llm_provider_from_request_query(request=request)
or await get_custom_llm_provider_from_request_body(request=request)
)
def _default_file_list_limit(custom_llm_provider: str | None) -> int | None:
return ANTHROPIC_DEFAULT_FILE_LIST_LIMIT if custom_llm_provider == LlmProviders.ANTHROPIC.value else None
def _deployment_model_mapping_keys(deployment: Deployment) -> frozenset[str]:
return frozenset(key for key in (deployment.model_name, deployment.model_info.id) if key is not None)
def _model_mapping_keys(deployments: Iterable[Deployment]) -> frozenset[str]:
return frozenset(itertools.chain.from_iterable(map(_deployment_model_mapping_keys, deployments)))
def _deployments_serving_model(llm_router: Router, model: str) -> tuple[Deployment, ...]:
deployment: Final = llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) else None
if deployment is not None:
return (deployment,)
return tuple(Deployment.model_validate(listed) for listed in llm_router.get_model_list(model_name=model) or ())
def _resolved_deployment_provider(deployment: Deployment) -> str | None:
try:
return litellm.get_llm_provider(
model=deployment.litellm_params.model,
custom_llm_provider=deployment.litellm_params.custom_llm_provider,
)[1]
except litellm.BadRequestError:
return None
def _deployments_on_provider(llm_router: Router | None, custom_llm_provider: str) -> Iterator[Deployment]:
if llm_router is None:
return iter(())
return (
deployment
for deployment_id in llm_router.get_model_ids()
if (deployment := llm_router.get_deployment(model_id=deployment_id)) is not None
and _resolved_deployment_provider(deployment) == custom_llm_provider
)
def _routed_model_name_on_provider(
llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
) -> bool:
if model_name in llm_router.model_names:
return model_name in provider_model_names
return llm_router.pattern_router.routes_to_provider(model_name, custom_llm_provider)
def _stored_model_name_routes_to_provider(
llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
) -> bool:
routing_group: Final = llm_router.get_routing_group(model_name)
routed_model_names: Final = (
tuple(routing_group.models) if routing_group is not None else (llm_router.routable_model_group(model_name),)
)
return any(
_routed_model_name_on_provider(llm_router, custom_llm_provider, provider_model_names, routed_model_name)
for routed_model_name in routed_model_names
)
def _files_on_provider(llm_router: Router | None, custom_llm_provider: str) -> _ListedFiles | _UnscopedFileList:
from litellm.proxy.pass_through_endpoints.managed_id_rewriter import passthrough_file_model_id
if custom_llm_provider not in litellm.provider_list:
return _UnscopedFileList(status_code=400, detail=f"Unknown provider '{custom_llm_provider}' for listing files.")
passthrough_model_id: Final = passthrough_file_model_id(custom_llm_provider)
deployments: Final = tuple(_deployments_on_provider(llm_router, custom_llm_provider))
model_mapping_keys: Final = _model_mapping_keys(deployments) | frozenset(
(passthrough_model_id,) if passthrough_model_id is not None else ()
)
if llm_router is None:
return _ListedFiles(FileListModelScope(model_mapping_keys), custom_llm_provider)
return _ListedFiles(
FileListModelScope(
model_mapping_keys,
routes_stored_model_name=functools.partial(
_stored_model_name_routes_to_provider,
llm_router,
custom_llm_provider,
frozenset(deployment.model_name for deployment in deployments),
),
),
custom_llm_provider,
)
def _files_serving_model(llm_router: Router, model: str) -> _ListedFiles:
deployments: Final = _deployments_serving_model(llm_router, model)
providers: Final = frozenset(map(_resolved_deployment_provider, deployments))
return _ListedFiles(
FileListModelScope(_model_mapping_keys(deployments)), next(iter(providers)) if len(providers) == 1 else None
)
async def _listed_files(
llm_router: Router | None,
user_api_key_dict: UserAPIKeyAuth,
custom_llm_provider: str | None,
target_model_names: str | None,
routed_model: str | None,
) -> _ListedFiles | _UnscopedFileList:
if routed_model is not None and llm_router is not None:
return _files_serving_model(llm_router, routed_model)
if target_model_names:
target_model_name: Final = _single_target_model_name(target_model_names)
if isinstance(target_model_name, _UnscopedFileList):
return target_model_name
if llm_router is None:
return _UnscopedFileList(
status_code=500, detail="LLM Router not initialized. Ensure models added to proxy."
)
await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=target_model_name,
user_api_key_dict=user_api_key_dict,
operation_context="file list",
)
return _files_serving_model(llm_router, target_model_name)
if custom_llm_provider is None:
return _ListedFiles(None, None)
return _files_on_provider(llm_router, custom_llm_provider)
@router.get(
"/{provider}/v1/files",
dependencies=[Depends(user_api_key_auth)],
@ -1697,6 +1883,8 @@ async def list_files(
purpose: str | None = None,
limit: int | None = None,
after: str | None = None,
after_id: str | None = None,
before_id: str | None = None,
):
"""
Returns information about a specific file. that can be used across - Assistants API, Batch API
@ -1739,6 +1927,10 @@ async def list_files(
)
response: Any | None = None
owned_files_hook: Final = proxy_logging_obj.get_proxy_hook("managed_files")
list_owned_files_only: Final = litellm.require_managed_files is True and not user_api_key_has_admin_view(
user_api_key_dict
)
# Check for model-based credential routing (no file_id encoding check for list)
should_route, model_used, _, credentials = await handle_model_based_routing(
@ -1750,7 +1942,36 @@ async def list_files(
check_file_id_encoding=False,
)
if should_route and credentials is not None:
if list_owned_files_only:
if after and after_id and after != after_id:
raise HTTPException(
status_code=400,
detail="Pass the list cursor as either 'after' or 'after_id', not both with different ids.",
)
if not isinstance(owned_files_hook, BaseFileEndpoints):
raise HTTPException(
status_code=500,
detail="Managed file listing is unavailable, so require_managed_files cannot scope the list to the caller.",
)
listed_files: Final = await _listed_files(
llm_router=llm_router,
user_api_key_dict=user_api_key_dict,
custom_llm_provider=await _requested_custom_llm_provider(request=request, provider=provider),
target_model_names=target_model_names,
routed_model=model_used if should_route else None,
)
if isinstance(listed_files, _UnscopedFileList):
_raise_unscoped_file_list(listed_files)
response = await _list_owned_managed_files( # rebind-ok: each dispatch branch binds response
managed_files_obj=owned_files_hook,
user_api_key_dict=user_api_key_dict,
purpose=purpose,
limit=limit if limit is not None else _default_file_list_limit(listed_files.custom_llm_provider),
after=after or after_id,
before=before_id,
model_scope=listed_files.model_scope,
)
elif should_route and credentials is not None:
# Use model-based routing with credentials from config
prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True)
response = await litellm.afile_list(
@ -1762,12 +1983,9 @@ async def list_files(
verbose_proxy_logger.debug("Listed files using model: %s", model_used)
elif target_model_names and isinstance(target_model_names, str):
target_model_names_list: Final = target_model_names.split(",")
if len(target_model_names_list) != 1:
raise HTTPException(
status_code=400,
detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
)
target_model_name: Final = _single_target_model_name(target_model_names)
if isinstance(target_model_name, _UnscopedFileList):
_raise_unscoped_file_list(target_model_name)
if llm_router is None:
raise HTTPException(
status_code=500,
@ -1775,7 +1993,7 @@ async def list_files(
)
credentials = await get_authorized_credentials_for_model(
llm_router=llm_router,
model_id=target_model_names_list[0],
model_id=target_model_name,
user_api_key_dict=user_api_key_dict,
operation_context="file list",
)
@ -1786,20 +2004,17 @@ async def list_files(
**data,
)
else:
custom_llm_provider: Final = (
provider
or get_custom_llm_provider_from_request_headers(request=request)
or get_custom_llm_provider_from_request_query(request=request)
or await get_custom_llm_provider_from_request_body(request=request)
)
custom_llm_provider: Final = await _requested_custom_llm_provider(request=request, provider=provider)
managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files")
if custom_llm_provider is None and isinstance(managed_files_obj, BaseFileEndpoints):
response = await managed_files_obj.afile_list(
purpose=purpose,
litellm_parent_otel_span=user_api_key_dict.parent_otel_span,
response = await _list_owned_managed_files( # rebind-ok: each dispatch branch binds response
managed_files_obj=managed_files_obj,
user_api_key_dict=user_api_key_dict,
purpose=purpose,
limit=limit,
after=after,
before=None,
model_scope=None,
)
else:
resolved_custom_llm_provider: Final = custom_llm_provider or "openai"

6
litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
View file

@ -242,6 +242,12 @@ def _passthrough_sentinel_model_id(provider: str) -> str:
return f"_passthrough_{provider}"
def passthrough_file_model_id(provider: str) -> str | None:
if (provider, "/v1/files") not in _LIST_ROUTE_TABLE:
return None
return _passthrough_sentinel_model_id(provider)
# Key under which the provider marker is stored in a file row's model_mappings.
# Its value lands in flat_model_file_ids (built from model_mappings.values()),
# giving the file table a DB-queryable provider scope it otherwise lacks.

33
litellm/proxy/proxy_server.py
View file

@ -14853,6 +14853,7 @@ async def _try_native_responses_count(
async def _provider_token_count(
count_with: Callable[[str, str | None], Awaitable[Union["TokenCountResponse", ProviderCountFailure, None]]],
model_to_use: str,
request_model: str,
settings: TokenCountSettings,
local_estimate_allowed: bool,
) -> Union["TokenCountResponse", ProviderCountFailure, None]:
@ -14875,17 +14876,28 @@ async def _provider_token_count(
if count_model is None:
return counted.model_copy(update={"count_source": "provider"})
return counted.model_copy(update={"count_source": "token_count_model", "model_used": count_model.model})
failure: Final = (
counted
if count_model is None or count_model.region is None
else counted.with_hint(
f"This model counts tokens with {count_model.model} in {count_model.region}. "
"If that region is not enabled for the AWS account, enable it or set token_count_model in model_info"
)
pinned_count_model: Final = None if count_model is None or count_model.region is None else count_model
region_hint: Final = (
None
if pinned_count_model is None
else f"This model counts tokens with {pinned_count_model.model} in {pinned_count_model.region}. "
"If that region is not enabled for the AWS account, enable it or set token_count_model in model_info"
)
if count_model is None and failure.cannot_count_model and settings.framing is not None and local_estimate_allowed:
return failure
raise failure.to_proxy_exception()
failure: Final = counted if region_hint is None else counted.with_hint(region_hint)
framing_can_stand_in: Final = (count_model is None or not count_model.configured) and (
failure.cannot_count_model or (pinned_count_model is not None and failure.denied_in_region)
)
if not (framing_can_stand_in and settings.framing is not None and local_estimate_allowed):
raise failure.to_proxy_exception()
if pinned_count_model is not None:
verbose_proxy_logger.warning(
"Token count for %s answered %s from %s in %s, so it falls back to the framed local estimate",
request_model,
failure.status_code,
pinned_count_model.model,
pinned_count_model.region,
)
return failure
@dataclass(frozen=True, slots=True)
@ -15455,6 +15467,7 @@ async def _count_request_tokens(
chat_params=MappingProxyType({}) if chat_count is None else chat_count.params,
),
model_to_use=model_to_use,
request_model=request.model,
settings=settings,
local_estimate_allowed=litellm.disable_token_counter is not True and contents_estimable,
)

28
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -4145,6 +4145,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "ZeroGPU",
"provider_display_name": "ZeroGPU",
"litellm_provider": "zerogpu",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://api.zerogpu.ai/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "zerogpu/gpt-oss-120b"
},
{
"provider": "CURSOR",
"provider_display_name": "Cursor",

3
litellm/proxy/route_llm_request.py
View file

@ -256,7 +256,6 @@ REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, ...]]] = Ma
"avideo_create_character": ("name", "video"),
"acreate_container": ("name",),
"aupload_container_file": ("file",),
"acreate_agent": ("name",),
"acreate_interaction": ("input",),
"acreate_eval": ("data_source_config", "testing_criteria"),
"acreate_run": ("data_source",),
@ -264,7 +263,7 @@ REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, ...]]] = Ma
)
REQUIRED_ONE_OF_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, str]]] = MappingProxyType(
{"acreate_interaction": ("model", "agent")}
{"acreate_interaction": ("model", "agent"), "acreate_agent": ("id", "name")}
)
JSON_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object])

71
litellm/proxy/schema.prisma
View file

@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
@@index([unified_file_id])
@@index([flat_model_file_ids], type: Gin)
@@index([team_id, created_at(sort: Desc)])
@@index([created_by, created_at(sort: Desc)])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
@@index([window_end])
@@index([created_at])
}
enum AccountingEventKind {
inbound_usage
outbound_usage
cost
}
enum AccountingDeliveryState {
pending
applied
parked
}
enum AccountingDeliveryTarget {
LiteLLM_VerificationToken
LiteLLM_UserTable
LiteLLM_TeamTable
LiteLLM_TeamMembership
LiteLLM_OrganizationTable
LiteLLM_OrganizationMembership
LiteLLM_EndUserTable
LiteLLM_AgentsTable
LiteLLM_TagTable
LiteLLM_ProjectTable
LiteLLM_ModelAccessGroupBudgetTable
LiteLLM_BudgetTable
LiteLLM_BudgetWindowSpend
LiteLLM_DailyUserSpend
LiteLLM_DailyTeamSpend
LiteLLM_DailyOrganizationSpend
LiteLLM_DailyEndUserSpend
LiteLLM_DailyAgentSpend
LiteLLM_DailyTagSpend
LiteLLM_DailyModelUsage
LiteLLM_DailyToolSpend
LiteLLM_AutoRouterDailySpend
}
// Durable accounting ledger. One row per usage or cost event; payload holds the
// private event envelope versioned by schema_version. Usage rows retain request-scoped
// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
model LiteLLM_AccountingEvent {
event_id String @id
kind AccountingEventKind
source_event_id String?
schema_version Int
payload_hash String
occurred_at DateTime
recorded_at DateTime @default(now())
payload Json
@@index([recorded_at])
@@index([kind, source_event_id])
}
// One row per (event, target table, version): pending work for the projector and,
// once applied in the same transaction as the target write, the receipt that stops a retry.
model LiteLLM_AccountingDelivery {
event_id String
target AccountingDeliveryTarget
version Int
state AccountingDeliveryState @default(pending)
attempts Int @default(0)
next_attempt_at DateTime @default(now())
applied_at DateTime?
last_error String?
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}

18
litellm/responses/main.py
View file

@ -2341,6 +2341,7 @@ async def acancel_responses(
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
litellm_metadata: Mapping[str, object] | None = None,
**kwargs,
) -> ResponsesAPIResponse:
"""
@ -2365,6 +2366,7 @@ async def acancel_responses(
cancel_responses,
response_id=response_id,
custom_llm_provider=custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
@ -2380,7 +2382,12 @@ async def acancel_responses(
response = await init_response
else:
response = init_response
return response
return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
responses_api_response=response,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
)
except Exception as e:
raise litellm.exception_type(
model=None,
@ -2402,6 +2409,7 @@ def cancel_responses(
timeout: float | httpx.Timeout | None = None,
# LiteLLM specific params,
custom_llm_provider: str | None = None,
litellm_metadata: Mapping[str, object] | None = None,
**kwargs,
) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
"""
@ -2471,7 +2479,13 @@ def cancel_responses(
shared_session=kwargs.get("shared_session"),
)
return response
if not isinstance(response, ResponsesAPIResponse):
return response
return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id(
responses_api_response=response,
litellm_metadata=litellm_metadata,
custom_llm_provider=custom_llm_provider,
)
except Exception as e:
raise litellm.exception_type(
model=None,

78
litellm/responses/streaming_iterator.py
View file

@ -41,7 +41,11 @@ from litellm.exceptions import MidStreamFallbackError, RateLimitError
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base
from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation
from litellm.litellm_core_utils.internal_call_metadata import (
is_background_response,
requests_background,
without_budget_reservation,
)
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
@ -65,7 +69,7 @@ from litellm.types.llms.openai import (
ResponsesAPIStreamingResponse,
)
from litellm.types.responses.main import GenericResponseOutputItem, OutputText
from litellm.types.utils import CallTypes
from litellm.types.utils import CallTypes, ModelResponse
from litellm.utils import async_post_call_success_deployment_hook
if TYPE_CHECKING:
@ -288,7 +292,7 @@ def stream_error_status_and_message(error_obj: object) -> tuple[int, str]:
return _status_code_for_error_fields(error_type, error_code), message
def _map_stream_error_to_exception(
def map_stream_error_to_exception(
error_obj: object,
model: str,
custom_llm_provider: str,
@ -578,6 +582,7 @@ class BaseResponsesAPIStreamingIterator:
self.request_data.get("input"),
self.request_data,
self._generated_content + self._generated_tool_arguments,
use_default_image_token_count=False,
)
if _estimate_wanted
else None
@ -731,7 +736,7 @@ class BaseResponsesAPIStreamingIterator:
self._earlier_rounds_spend = earlier
def _map_error_event_exception(self, error_obj: object) -> Exception:
return _map_stream_error_to_exception(
return map_stream_error_to_exception(
error_obj,
self.model or "",
self.custom_llm_provider or "",
@ -981,6 +986,39 @@ class BaseResponsesAPIStreamingIterator:
except Exception:
pass
def _record_partial_usage_for_failure(self) -> None:
generated_text: Final = self._generated_content + self._generated_tool_arguments
if (
not generated_text
or requests_background(self.request_data)
or is_background_response(self._created_response)
or self._completed_response_logged
or self.logging_obj.model_call_details.get("combined_usage_object") is not None
):
return
estimate: Final = _estimate_usage_safely(
self.model or "",
self.request_data.get("input"),
self.request_data,
generated_text,
use_default_image_token_count=True,
)
if estimate is None:
return
usage: Final = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage( # pyright: ignore[reportPrivateUsage] # same shared transform the success path above uses
estimate
)
served_tier: Final[object] = getattr(self._created_response, "service_tier", None)
assembled: Final = ModelResponse(
model=self.model, usage=usage, service_tier=served_tier if isinstance(served_tier, str) else None
)
if self._earlier_rounds_spend is None:
self.logging_obj.record_assembled_response_for_failure(assembled)
return
self.logging_obj.record_failure_after_earlier_rounds(
self._earlier_rounds_spend, usage, self.logging_obj.response_cost_calculator(result=assembled) or 0.0
)
def _handle_failure(self, exception: Exception):
"""
Trigger failure handlers before bubbling the exception.
@ -990,6 +1028,7 @@ class BaseResponsesAPIStreamingIterator:
if self._failure_handled:
return
self._failure_handled = True
self._record_partial_usage_for_failure()
traceback_exception: Final = traceback.format_exc()
end_time: Final = datetime.now()
@ -1060,15 +1099,20 @@ class BaseResponsesAPIStreamingIterator:
the same shape the success handler assembles for a natural end of stream.
Its response is the response.created payload (it carries the served
service_tier) plus the text and tool arguments relayed so far, with counted
usage since the provider never sent any. None when nothing was relayed yet
or the stream already reached its terminal event and billed itself.
usage since the provider never sent any. None when nothing was relayed yet,
the stream already reached its terminal event and billed itself, or its failure
log already billed the relayed output.
"""
created: Final = self._created_response
generated: Final = self._generated_content + self._generated_tool_arguments
if self.completed_response is not None or created is None or not generated:
if created is None or not generated or self._billed_without_disconnect():
return None
usage: Final = _estimate_usage_safely(
self.model or "", self.request_data.get("input"), self.request_data, generated
self.model or "",
self.request_data.get("input"),
self.request_data,
generated,
use_default_image_token_count=False,
)
if usage is None:
return None
@ -1082,6 +1126,9 @@ class BaseResponsesAPIStreamingIterator:
stamp_responses_usage_cost(partial, self.logging_obj)
return ResponseIncompleteEvent(type=ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE, response=partial)
def _billed_without_disconnect(self) -> bool:
return self.completed_response is not None or self._failure_handled
def _note_yielded_event(self, event: ResponsesAPIStreamingResponse) -> None:
self._yielded_first_chunk = True
if event.type not in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES:
@ -1218,6 +1265,9 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
raise StopAsyncIteration from e
self._raise_for_transport_error(e)
except httpx.TimeoutException as e:
if self.completed_response is not None:
self.finished = True
raise StopAsyncIteration from e
self._raise_for_transport_timeout(e)
except httpx.HTTPError as e:
# Handle HTTP errors
@ -1303,6 +1353,9 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
raise StopIteration from e
self._raise_for_transport_error(e)
except httpx.TimeoutException as e:
if self.completed_response is not None:
self.finished = True
raise StopIteration from e
self._raise_for_transport_timeout(e)
except httpx.HTTPError as e:
# Handle HTTP errors
@ -1727,6 +1780,8 @@ def _estimate_usage_from_text(
request_input: object,
responses_api_request: Mapping[str, object],
generated_text: str,
*,
use_default_image_token_count: bool = False,
) -> ResponseAPIUsage:
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped
input=request_input, # pyright: ignore[reportArgumentType] # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union
@ -1734,7 +1789,7 @@ def _estimate_usage_from_text(
only_since_compaction=True,
)
input_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped
model=model, messages=messages
model=model, messages=messages, use_default_image_token_count=use_default_image_token_count
)
output_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped
model=model, text=generated_text, count_response_tokens=True
@ -1751,6 +1806,8 @@ def _estimate_usage_safely(
request_input: object,
responses_api_request: Mapping[str, object],
generated_text: str,
*,
use_default_image_token_count: bool,
) -> ResponseAPIUsage | None:
try:
return _estimate_usage_from_text(
@ -1758,6 +1815,7 @@ def _estimate_usage_safely(
request_input=request_input,
responses_api_request=responses_api_request,
generated_text=generated_text,
use_default_image_token_count=use_default_image_token_count,
)
except Exception as e:
verbose_logger.debug("Could not estimate usage from stream text, billing $0: %s", e)
@ -2276,7 +2334,7 @@ class _ResponsesWsTurn:
)
if failed_event is None:
return None
return _map_stream_error_to_exception(
return map_stream_error_to_exception(
_ws_event_error(failed_event), model, custom_llm_provider, logging_obj=connection_logging_obj
)

109
litellm/responses/utils.py
View file

@ -1,8 +1,10 @@
import base64
import os
import re
from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
from functools import reduce
from typing import Any, Final, Optional, TypeVar, Union, cast, get_type_hints, overload
from types import MappingProxyType
from typing import Any, Final, Optional, TypeVar, Union, cast, get_args, get_type_hints, overload
from pydantic import BaseModel, TypeAdapter
from typing_extensions import TypedDict as ExtraItemsTypedDict
@ -14,6 +16,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.openai import (
AllMessageValues,
InputTokensDetails,
@ -29,6 +32,7 @@ from litellm.types.responses.main import DecodedResponseId
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
ModelInfo,
PromptTokensDetailsWrapper,
ProviderSpecificHeader,
SpecialEnums,
@ -399,7 +403,7 @@ class ResponsesAPIRequestUtils:
def update_responses_api_response_id_with_model_id(
responses_api_response: ResponsesAPIResponse,
custom_llm_provider: str | None,
litellm_metadata: dict[str, object] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
) -> ResponsesAPIResponse:
...
@ -408,7 +412,7 @@ class ResponsesAPIRequestUtils:
def update_responses_api_response_id_with_model_id(
responses_api_response: dict[str, object],
custom_llm_provider: str | None,
litellm_metadata: dict[str, object] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
) -> dict[str, object]:
...
@ -418,7 +422,7 @@ class ResponsesAPIRequestUtils:
def update_responses_api_response_id_with_model_id(
responses_api_response: ResponsesAPIResponse | dict[str, Any],
custom_llm_provider: str | None,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, Any] | None = None,
) -> ResponsesAPIResponse | dict[str, object]:
"""Update the responses_api_response_id with model_id and custom_llm_provider.
@ -1110,7 +1114,7 @@ class ResponsesAPIRequestUtils:
def _update_container_ids_in_response(
responses_api_response: ResponsesAPIResponse | dict[str, object],
custom_llm_provider: str | None,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, Any] | None = None,
) -> ResponsesAPIResponse | dict[str, object]:
"""Encode container IDs in the response output with provider/model info.
@ -1382,3 +1386,98 @@ def auto_reasoning_summary(model: str, custom_llm_provider: object) -> Reasoning
provider, model.removeprefix(f"{provider}/")
)
return "detailed" if responses_config is None else responses_config.auto_reasoning_summary()
def opt_in_auto_reasoning_summary(model: str | None, custom_llm_provider: object) -> ReasoningSummaryLevel | None:
if not (litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"):
return None
return "detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider)
def reasoning_with_summary_default(
reasoning: Mapping[str, object], summary_alias: object, model: str, custom_llm_provider: object
) -> Mapping[str, object]:
if "summary" in reasoning:
return {**reasoning}
summary: Final = (
summary_alias if summary_alias is not None else opt_in_auto_reasoning_summary(model, custom_llm_provider)
)
return {**reasoning} if summary is None else {**reasoning, "summary": summary}
_SUMMARY_VALUES: Final = frozenset((*get_args(ReasoningSummaryLevel), None))
def _openai_model_info(model: str) -> ModelInfo | None:
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
model_info: Final = lookup_model_info_for_registration(model.removeprefix("responses/"), "openai")
return None if isinstance(model_info, ModelInfoLookupMiss) else model_info
def _declared_values(values: Sequence[str] | None) -> frozenset[str | None] | None:
return None if values is None else frozenset(values)
def _openai_reasoning_field_values(model: str) -> Mapping[str, frozenset[str | None] | None]:
model_info: Final = _openai_model_info(model)
if model_info is None:
return MappingProxyType({"summary": _SUMMARY_VALUES, "mode": None, "context": None})
if not model_info.get("supports_reasoning"):
return MappingProxyType({})
return MappingProxyType(
{
"summary": _SUMMARY_VALUES,
"mode": _declared_values(model_info.get("reasoning_modes")),
"context": _declared_values(model_info.get("reasoning_contexts")),
}
)
def _openai_accepts_reasoning_field(
key: str, value: object, values_by_field: Mapping[str, frozenset[str | None] | None]
) -> bool:
if key == "effort":
return True
if key not in values_by_field:
return False
allowed: Final = values_by_field[key]
return allowed is None or (isinstance(value, str | None) and value in allowed)
def _openai_accepted_reasoning(reasoning: Mapping[str, object], model: str) -> Mapping[str, object]:
values_by_field: Final = _openai_reasoning_field_values(model)
return {
key: value for key, value in reasoning.items() if _openai_accepts_reasoning_field(key, value, values_by_field)
}
def _additional_drop_paths(additional_drop_params: object) -> tuple[str, ...]:
if additional_drop_params.__class__ is not list:
return ()
return tuple(path for path in iter_items(additional_drop_params) if isinstance(path, str))
def openai_bridge_reasoning(
*,
requested: Mapping[str, object],
effort_kept: bool,
summary_alias: object,
model: str,
drop_params: bool,
additional_drop_params: object,
) -> Mapping[str, object] | None:
drop_paths: Final = _additional_drop_paths(additional_drop_params)
if "reasoning_effort" in drop_paths:
return None
kept: Final = requested if effort_kept else {key: value for key, value in requested.items() if key != "effort"}
with_summary: Final = reasoning_with_summary_default(kept, summary_alias, model, "openai")
accepted: Final = (
_openai_accepted_reasoning(with_summary, model)
if drop_params and OpenAIGPTConfig.is_openai_catalog_model(model)
else {**with_summary}
)
remaining: Final = as_str_mapping(
_apply_nested_drop_params({"reasoning_effort": accepted}, list(drop_paths)).get("reasoning_effort")
)
return remaining or None

190
litellm/router.py
View file

@ -216,6 +216,8 @@ from litellm.router_utils.common_utils import (
deployments_the_caller_may_use,
filter_team_based_models,
filter_web_search_deployments,
format_content_policy_fallback_hint,
format_content_policy_no_fallback_note,
format_fallback_outcome_message,
format_no_fallback_group_message,
get_request_team_id,
@ -631,6 +633,7 @@ _DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLite
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
_DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected"
_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
_DEPLOYMENTS_ADAPTER: Final = TypeAdapter(Sequence[Mapping[str, object]])
def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]:
@ -692,6 +695,12 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
def _every_deployment_refused(
deployments: Sequence[Mapping[str, object]], refused_deployment_ids: tuple[str, ...]
) -> bool:
return all(_configured_model_info(deployment).get("id") in refused_deployment_ids for deployment in deployments)
def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
if silent_model is None:
return ()
@ -832,6 +841,10 @@ def _request_model_group(kwargs: Mapping[str, object]) -> str | None:
return model_group if isinstance(model_group, str) else None
def _mid_stream_error_before_output(error: "MidStreamFallbackError") -> bool:
return not error.generated_content or error.is_pre_first_chunk
def _mid_stream_retry_trigger(error: "MidStreamFallbackError") -> Exception:
"""The provider's own error, which is what the retry policy and should_retry_this_error classify."""
return error.original_exception if error.original_exception is not None else error
@ -962,6 +975,10 @@ def _responses_stream_holds_event(item: object, held_event_count: int) -> bool:
return getattr(item, "type", None) in PRE_OUTPUT_LIFECYCLE_EVENT_TYPES
def _responses_event_announces_background_run(item: object) -> bool:
return getattr(getattr(item, "response", None), "background", None) is True
class FallbackAwareAnthropicMessagesStream:
"""
Bare async generators can't carry the `_hidden_params` attribute the
@ -4127,11 +4144,16 @@ class Router:
async def stream_with_fallbacks():
held_lifecycle_events: tuple[object, ...] = () # rebind-ok: flushed at first output, dropped on fallback
response_id_exposed = False # rebind-ok: set once, before an early-released lifecycle event is yielded
try:
async for item in cast( # cast-ok: response iterators are async streams
AsyncIterator[object], source_iterator
):
if _responses_stream_holds_event(item, len(held_lifecycle_events)):
if not response_id_exposed and self._aresponses_background_stream_can_release(
item, source_iterator, initial_kwargs
):
response_id_exposed = True
if not response_id_exposed and _responses_stream_holds_event(item, len(held_lifecycle_events)):
held_lifecycle_events = (*held_lifecycle_events, item)
continue
for held_event in held_lifecycle_events:
@ -4141,6 +4163,8 @@ class Router:
for held_event in held_lifecycle_events:
yield held_event
except MidStreamFallbackError as e:
if response_id_exposed:
self._raise_released_stream_error(e, initial_kwargs)
async with contextlib.aclosing(
self._aresponses_fallback_attempt(
e, source_iterator, initial_kwargs, wrapper.adopt_fallback_headers, held_lifecycle_events
@ -4166,6 +4190,54 @@ class Router:
wrapper: Final = FallbackResponsesStreamWrapper(stream_with_fallbacks())
return wrapper
def _aresponses_background_stream_can_release(
self,
item: object,
source_iterator: "BaseResponsesAPIStreamingIterator",
kwargs: Mapping[str, object],
) -> bool:
from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator
if (
type(source_iterator) is not ResponsesAPIStreamingIterator
or source_iterator.custom_llm_provider != "openai"
):
return False
if not _responses_event_announces_background_run(item):
return False
model_group: Final = kwargs.get("model")
return isinstance(model_group, str) and not self._stream_can_fall_back(model_group, kwargs)
def _raise_released_stream_error(
self, e: "MidStreamFallbackError", initial_kwargs: Mapping[str, object]
) -> NoReturn:
original: Final = e.original_exception
model_group: Final = initial_kwargs.get("model")
if (
litellm.expose_router_debug_in_errors
and isinstance(original, litellm.ContentPolicyViolationError)
and isinstance(model_group, str)
and not fallbacks_disabled_for_request(initial_kwargs)
and _mid_stream_error_before_output(e)
):
hop_depth: Final = initial_kwargs.get("fallback_depth")
original.message += format_content_policy_no_fallback_note(
model_group,
fallback_lookup_groups(initial_kwargs, model_group),
_request_fallback_list(
initial_kwargs,
"content_policy_fallbacks",
cast("list[object] | None", self.content_policy_fallbacks), # cast-ok: untyped router attribute
),
_request_fallback_list(
initial_kwargs,
"fallbacks",
cast("list[object] | None", self.fallbacks), # cast-ok: untyped router attribute
),
isinstance(hop_depth, int) and hop_depth > 0,
)
Router._raise_original_mid_stream_error(e)
def _responses_continuation(
self,
e: "MidStreamFallbackError",
@ -4175,7 +4247,7 @@ class Router:
"""What a fallback continues from once a Responses stream broke after text, raising the primary's error
when the request constrains its output or the stream began a tool call, since a continued JSON document or
tool call would not parse, and when a content filter cut it, so no other model finishes filtered text."""
if not e.generated_content or e.is_pre_first_chunk:
if _mid_stream_error_before_output(e):
return None
if source_iterator.started_tool_call or isinstance(e.original_exception, litellm.ContentPolicyViolationError):
Router._raise_original_mid_stream_error(e)
@ -4706,7 +4778,13 @@ class Router:
@staticmethod
def _stamp_retry_skip_deployment_id(exception: Exception, kwargs: Mapping[str, object]) -> None:
effective_model_info: Final = kwargs.get("model_info")
deployment_id: Final = effective_model_info.get("id") if isinstance(effective_model_info, Mapping) else None
if not isinstance(effective_model_info, Mapping):
return
model_info: Final = _MODEL_INFO_ADAPTER.validate_python(effective_model_info)
configured_deployment_id: Final = (
model_info.get("original_model_id") if isinstance(exception, litellm.UnsupportedEndpointError) else None
)
deployment_id: Final = configured_deployment_id or model_info.get("id")
if isinstance(deployment_id, str) and deployment_id:
exception.retry_skip_deployment_id = deployment_id # pyright: ignore[reportAttributeAccessIssue] # dynamic stamp, read by _deployment_ids_to_skip_on_retry
@ -6378,6 +6456,9 @@ class Router:
hop_kwargs: Final = mid_stream_fallback_hop_kwargs(
model=model, original_generic_function=original_generic_function, controls=controls, kwargs=kwargs
)
shared_logging: Final = kwargs.get("litellm_logging_obj")
if isinstance(shared_logging, LiteLLMLogging):
shared_logging.carry_failed_attempt_usage()
response: Final = await self._ageneric_api_call_with_fallbacks_helper(
model=model, original_generic_function=original_generic_function, **kwargs
)
@ -6487,7 +6568,7 @@ class Router:
model: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
has_generated_content = not ( # rebind-ok: set once real content is seen, the buffer cap is hit, or neither a retry nor a fallback can take over
self._anthropic_messages_stream_can_retry(initial_kwargs)
or self._anthropic_messages_stream_can_fall_back(model, initial_kwargs)
or self._stream_can_fall_back(model, initial_kwargs)
)
buffered_lifecycle_chunks: tuple[bytes, ...] = () # rebind-ok: flushed once committed or on decline
try:
@ -6710,7 +6791,7 @@ class Router:
budget, _ = self._stream_retry_budget(frame_error, kwargs)
if budget > attempted_retries_for_request(kwargs):
return frame_error
if self._anthropic_messages_stream_can_fall_back(model_group, kwargs):
if self._stream_can_fall_back(model_group, kwargs):
return frame_error
return None
@ -8725,7 +8806,9 @@ class Router:
)
return response
else:
error_message = f"model={model_group}. content_policy_fallback={mask_sensitive_structure(content_policy_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks"
error_message = format_content_policy_fallback_hint(
model_group, content_policy_fallbacks, fallbacks
)
verbose_router_logger.info(
msg=f"Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \
to fallbacks, if available.{error_message}"
@ -9048,6 +9131,10 @@ class Router:
)
if first_skipped_ids:
kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it
if isinstance(original_exception, litellm.UnsupportedEndpointError) and _every_deployment_refused(
_DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), first_skipped_ids
):
raise
else:
raise
@ -9127,6 +9214,10 @@ class Router:
)
if skipped_ids:
kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it
if isinstance(e, litellm.UnsupportedEndpointError) and _every_deployment_refused(
_DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids
):
raise
_timeout = self._time_to_sleep_before_retry(
e=e,
remaining_retries=remaining_retries,
@ -9223,7 +9314,7 @@ class Router:
status_code: Final = getattr(error, "status_code", None)
if status_code is not None and not litellm.should_retry(status_code):
# 401/403 are special cases - allow retry if multiple deployments exist (handled below)
if status_code not in (401, 403):
if status_code not in (401, 403) and not isinstance(error, litellm.UnsupportedEndpointError):
raise error
if isinstance(error, litellm.NotFoundError):
@ -9873,7 +9964,7 @@ class Router:
)
return has_unattempted_fallback_target(resolved, kwargs)
def _anthropic_messages_order_levels(self, model_group: str, kwargs: Mapping[str, Any]) -> tuple[int, ...]:
def _stream_order_levels(self, model_group: str, kwargs: Mapping[str, Any]) -> tuple[int, ...]:
"""
The distinct deployment order levels the fallback dispatcher would see for this request,
computed the same way: the tier a pre-routing hook selected wins over the requested group.
@ -9891,7 +9982,7 @@ class Router:
)
)
def _anthropic_messages_stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
def _stream_can_fall_back(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
"""
Whether async_function_with_fallbacks_common_utils could still route a
MidStreamFallbackError somewhere for this request (order levels, weighted
@ -9903,7 +9994,7 @@ class Router:
return False
if self.enable_weighted_failover:
return True
order_levels: Final = self._anthropic_messages_order_levels(model_group, kwargs)
order_levels: Final = self._stream_order_levels(model_group, kwargs)
if len(order_levels) > 1:
current_target: Final = kwargs.get("_target_order")
skip_up_to: Final = current_target if current_target is not None else order_levels[0]
@ -10276,28 +10367,38 @@ class Router:
model_info[field] = backend_value
@staticmethod
def _cost_map_backend_model(deployment: Deployment) -> str:
model_info_base_model: Final = deployment.model_info.base_model
def _cost_map_backend_model(deployment: Deployment | DeploymentTypedDict) -> str:
model_info_base_model: Final = (
deployment.model_info.base_model
if isinstance(deployment, Deployment)
else _configured_model_info(deployment).get("base_model")
)
if isinstance(model_info_base_model, str) and model_info_base_model:
return model_info_base_model
params_base_model: Final = deployment.litellm_params.get("base_model")
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(
deployment.litellm_params.model_dump(exclude_none=True)
if isinstance(deployment, Deployment)
else deployment["litellm_params"]
)
params_base_model: Final = params.get("base_model")
if isinstance(params_base_model, str) and params_base_model:
return params_base_model
params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment.litellm_params.model_dump(exclude_none=True))
model: Final = (
deployment.litellm_params.model
if isinstance(deployment, Deployment)
else deployment["litellm_params"].get("model", "")
)
invoked_model: Final = params.get("model_id")
if (
isinstance(invoked_model, str)
and invoked_model
and (
deployment.litellm_params.custom_llm_provider == "bedrock"
or deployment.litellm_params.model.startswith("bedrock/")
)
and (params.get("custom_llm_provider") == "bedrock" or model.startswith("bedrock/"))
):
profile_model: Final = invoked_model if invoked_model.startswith("bedrock/") else f"bedrock/{invoked_model}"
profile_info: Final = lookup_model_info_for_registration(model=profile_model, custom_llm_provider="bedrock")
if not isinstance(profile_info, ModelInfoLookupMiss):
return profile_model
return deployment.litellm_params.model
return model
@staticmethod
def _inherit_builtin_service_tier_pricing(
@ -13857,7 +13958,9 @@ class Router:
]
return named or self._get_wildcard_deployments(model_name=model_group)
def resolved_litellm_models(self, model_name: str, team_id: str | None = None) -> tuple[str, ...]:
def resolved_litellm_models(
self, model_name: str, team_id: str | None = None, *, use_base_model: bool = False
) -> tuple[str, ...]:
"""The provider model strings `model_name` can actually be served by on this proxy.
`get_model_list` composes every channel the request path itself uses (exact name,
@ -13870,6 +13973,12 @@ class Router:
deployment behind it - so the fallback for an empty result is the caller's policy,
never this function's.
"""
if use_base_model:
return tuple(
backend_model
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
if (backend_model := self._cost_map_backend_model(deployment))
)
return tuple(
litellm_model
for deployment in self.get_model_list(model_name=model_name, team_id=team_id) or ()
@ -14439,7 +14548,6 @@ class Router:
Filter out model in model group, if:
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- [TODO] function call and model doesn't support function calling
"""
@ -14454,8 +14562,6 @@ class Router:
_context_window_error = False
_potential_error_str = ""
_rate_limit_error = False
parent_otel_span: Final = get_parent_otel_span_from_kwargs(request_kwargs)
has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending(
request_kwargs
@ -14479,13 +14585,6 @@ class Router:
else frozenset()
)
## get model group RPM ##
dt: Final = get_utc_datetime()
current_minute: Final = dt.strftime("%H-%M")
rpm_key: Final = f"{model}:rpm:{current_minute}"
model_group_cache: Final = (
self.cache.get_cache(key=rpm_key, local_only=True, parent_otel_span=parent_otel_span) or {}
) # check the in-memory cache used by lowest_latency and usage-based routing. Only check the local cache.
for idx, deployment in enumerate(_returned_deployments):
# Cache nested dict access to avoid repeated temporary dict allocations
_litellm_params = deployment.get("litellm_params", {})
@ -14508,24 +14607,6 @@ class Router:
)
continue
model_id = _model_info.get("id", "")
## RPM CHECK ##
### get local router cache ###
current_request_cache_local = (
self.cache.get_cache(key=model_id, local_only=True, parent_otel_span=parent_otel_span) or 0
)
### get usage based cache ###
if isinstance(model_group_cache, dict) and self.routing_strategy != "usage-based-routing-v2":
model_group_cache[model_id] = model_group_cache.get(model_id, 0)
current_request = max(current_request_cache_local, model_group_cache[model_id])
if isinstance(_litellm_params, dict) and _litellm_params.get("rpm", None) is not None:
if isinstance(_litellm_params["rpm"], int) and _litellm_params["rpm"] <= current_request:
invalid_model_indices.add(idx)
_rate_limit_error = True
continue
## REGION CHECK ##
if request_kwargs is not None and request_kwargs.get("allowed_model_region") is not None:
allowed_model_region = request_kwargs.get("allowed_model_region")
@ -14562,12 +14643,6 @@ class Router:
)
if len(excluded_indices) == len(_returned_deployments):
"""
- no healthy deployments available b/c context window checks or rate limit error
- First check for rate limit errors (if this is true, it means the model passed the context window check but failed the rate limit check)
"""
group_deployments: Final = (
tuple(request_deployments)
if request_deployments is not None
@ -14589,12 +14664,7 @@ class Router:
),
model=model,
)
if _rate_limit_error is True: # allow generic fallback logic to take place
raise RouterRateLimitErrorBasic(
model=model,
)
elif _context_window_error is True:
if _context_window_error is True:
raise litellm.ContextWindowExceededError(
message=f"litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.\n{_potential_error_str}",
model=model,

37
litellm/router_utils/common_utils.py
View file

@ -39,6 +39,9 @@ _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] =
tuple[Mapping[str, object], ...]
)
_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
_FALLBACK_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
tuple[Mapping[str, object], ...]
)
def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment:
@ -256,6 +259,40 @@ def format_fallback_outcome_message(
return f"{lead} Fallback model group(s) configured: {targets}."
def format_content_policy_fallback_hint(
model_group: str | None, content_policy_fallbacks: object, fallbacks: object
) -> str:
return (
f"model={model_group}. content_policy_fallback={mask_sensitive_structure(content_policy_fallbacks)}. "
f"fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'content_policy_fallback' - "
"https://docs.litellm.ai/docs/routing#fallbacks"
)
def _as_fallback_entries(fallbacks: object) -> tuple[Mapping[str, object], ...] | None:
try:
return _FALLBACK_ENTRIES_ADAPTER.validate_python(fallbacks)
except ValidationError:
return None
def format_content_policy_no_fallback_note(
model_group: str,
lookup_groups: tuple[str, ...],
content_policy_fallbacks: object,
fallbacks: object,
nested_fallback_hop: bool,
) -> str:
outcome: Final = "" if nested_fallback_hop else format_fallback_outcome_message(model_group, None, "")
if content_policy_fallbacks is not None:
return outcome
hint: Final = f"\n{format_content_policy_fallback_hint(model_group, content_policy_fallbacks, fallbacks)}"
fallback_entries: Final = _as_fallback_entries(fallbacks) if fallbacks is not None else None
if fallback_entries is None or not lookup_groups or nested_fallback_hop:
return hint + outcome
return hint + format_no_fallback_group_message(lookup_groups, fallback_entries)
def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:
"""
Hash of the credential params, used for mapping the file id to the right model

2
litellm/router_utils/cooldown_handlers.py
View file

@ -356,7 +356,7 @@ def _should_run_cooldown_logic(
)
return False
if isinstance(original_exception, litellm.MalformedRequestFieldError):
if isinstance(original_exception, (litellm.MalformedRequestFieldError, litellm.UnsupportedEndpointError)):
verbose_router_logger.debug(
"Should Not Run Cooldown Logic: litellm rejected the request before calling the deployment"
)

5
litellm/router_utils/fallback_event_handlers.py
View file

@ -261,7 +261,10 @@ def is_bad_request_without_fallback(exception: Exception, fallback_on_unsupporte
return False
if isinstance(exception, litellm.UnsupportedParamsError):
return True
if isinstance(exception, (litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError)):
if isinstance(
exception,
(litellm.ContextWindowExceededError, litellm.ContentPolicyViolationError, litellm.UnsupportedEndpointError),
):
return False
return exception.status_code == 400 and bool(getattr(exception, "llm_provider", ""))

3
litellm/router_utils/get_retry_from_policy.py
View file

@ -13,6 +13,7 @@ from litellm.exceptions import (
RateLimitError,
ServiceUnavailableError,
Timeout,
UnsupportedEndpointError,
)
from litellm.types.router import RetryPolicy
@ -59,6 +60,8 @@ def get_num_retries_from_retry_policy(
policy: Final = resolve_retry_policy(retry_policy, model_group, model_group_retry_policy)
if policy is None:
return None
if isinstance(exception, UnsupportedEndpointError):
return policy.DefaultRetries
by_class: Final = (
_RETRIES_BY_EXCEPTION_TYPE[cls](policy) for cls in type(exception).__mro__ if cls in _RETRIES_BY_EXCEPTION_TYPE
)

6
litellm/rust_bridge/AGENTS.md
View file

@ -46,6 +46,12 @@
- A failure after execution starts is terminal and never triggers fallback
- Creating an unstarted native coroutine acquires no clients or credentials and captures no execution context
# Native extension
- `_native` ships in every wheel and the sdist builds it, so it is always importable
- Import it with `from litellm.rust_bridge import _native` and use the attributes `_native.pyi` declares
- Never add a `None` branch, `getattr` default or stand-in class for a missing extension
# Verification
- Check cross-language behavior against a freshly installed extension, covering unstarted-call release, setup ordering, exception identity, sync and async streams, cancellation and GC

7
litellm/rust_bridge/__init__.py
View file

@ -1,10 +1,5 @@
"""LiteLLM Rust bridge package."""
from litellm.rust_bridge.configuration import rust
from litellm.rust_bridge.loader import (
get_native_bridge,
native_bridge_available,
reset_native_bridge_cache,
)
__all__ = ["get_native_bridge", "native_bridge_available", "reset_native_bridge_cache", "rust"]
__all__ = ["rust"]

26
litellm/rust_bridge/bindings.py
View file

@ -3,7 +3,7 @@ from __future__ import annotations
from collections.abc import Callable
from typing import Final, Generic, TypeVar
from litellm.rust_bridge.loader import get_native_bridge
from litellm.rust_bridge import _native
BindingT = TypeVar("BindingT")
@ -18,32 +18,24 @@ _UNSET: Final = _Unset()
class NativeBinding(Generic[BindingT]):
"""Resolve one native attribute with an explicit, resettable test override."""
def __init__(self, attribute: str, *, validate: Callable[[object], BindingT | None]) -> None:
def __init__(
self, attribute: str, *, validate: Callable[[object], BindingT | None], source: object = _native
) -> None:
self._attribute: Final = attribute
self._validate: Final = validate
self._source: Final = source
self._override: BindingT | None | _Unset = _UNSET
def load(self) -> BindingT | None:
if not isinstance(self._override, _Unset):
return self._override
native: Final = get_native_bridge()
if native is None:
return None
return self._validate(getattr(native, self._attribute, None))
return self._validate(getattr(self._source, self._attribute, None))
def with_source(self, source: object) -> NativeBinding[BindingT]:
return NativeBinding(self._attribute, validate=self._validate, source=source)
def override(self, value: BindingT | None) -> None:
self._override = value
def reset(self) -> None:
self._override = _UNSET
def native_exception_types() -> tuple[type[BaseException], type[BaseException]] | None:
native: Final = get_native_bridge()
if native is None:
return None
declined: Final = getattr(native, "RustBridgeDeclined", None)
upstream: Final = getattr(native, "RustUpstreamError", None)
if not isinstance(declined, type) or not isinstance(upstream, type):
return None
return declined, upstream

37
litellm/rust_bridge/clickhouse.py
View file

@ -1,12 +1,10 @@
import os
from collections.abc import Awaitable, Mapping, Sequence
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Final, Protocol, runtime_checkable
from pydantic import ConfigDict, TypeAdapter
from typing import Final
from litellm.constants import DEFAULT_AGENT_TRACING_RETENTION_DAYS, DEFAULT_CLICKHOUSE_DATABASE
from litellm.rust_bridge.loader import get_native_bridge
from litellm.rust_bridge import _native
@dataclass(frozen=True, slots=True, repr=False)
@ -16,27 +14,6 @@ class SpendStorageConfig:
retention_days: int = DEFAULT_AGENT_TRACING_RETENTION_DAYS
class NativeSpendConfig(Protocol):
def __init__(self, database: str, url: str, retention_days: int) -> None: ...
class NativeSpendStorage(Protocol):
def __init__(self, config: NativeSpendConfig) -> None: ...
def ensure_schema(self) -> Awaitable[None]: ...
def insert_rows(self, rows: Sequence[Mapping[str, object]]) -> Awaitable[None]: ...
@runtime_checkable
class NativeSpendBridge(Protocol):
NativeClickHouseSpendConfig: type[NativeSpendConfig]
NativeClickHouseSpendStorage: type[NativeSpendStorage]
_NATIVE: Final[TypeAdapter[NativeSpendBridge]] = TypeAdapter(
NativeSpendBridge, config=ConfigDict(arbitrary_types_allowed=True)
)
def spend_storage_config(environ: Mapping[str, str] = os.environ) -> SpendStorageConfig:
url: Final = environ.get("CLICKHOUSE_URL")
if not url:
@ -53,12 +30,8 @@ def spend_storage_config(environ: Mapping[str, str] = os.environ) -> SpendStorag
class ClickHouseSpendStorage:
def __init__(self, config: SpendStorageConfig) -> None:
bridge: Final = get_native_bridge()
if bridge is None:
raise RuntimeError("ClickHouse spend logging requires the Rust extension")
native: Final = _NATIVE.validate_python(bridge)
self._native: Final = native.NativeClickHouseSpendStorage(
native.NativeClickHouseSpendConfig(config.database, config.url, config.retention_days)
self._native: Final = _native.NativeClickHouseSpendStorage(
_native.NativeClickHouseSpendConfig(config.database, config.url, config.retention_days)
)
async def ensure_schema(self) -> None:

20
litellm/rust_bridge/fork_guard.py
View file

@ -11,32 +11,22 @@ site, so the runtime can never start there. Workers forked from it are unaffecte
from __future__ import annotations
from collections.abc import Callable
from typing import Final
from litellm.rust_bridge.loader import get_native_bridge
from litellm.rust_bridge import _native
class NativeStateStartedBeforeFork(RuntimeError):
pass
class _NeverRaised(RuntimeError):
"""Stands in for a native exception when the extension is unavailable or predates it."""
ForkedAfterNativeRuntimeStarted: Final = _native.ForkedAfterNativeRuntimeStarted
ProcessReservedForForking: Final = _native.ProcessReservedForForking
_native: Final = get_native_bridge()
ForkedAfterNativeRuntimeStarted: Final[type[RuntimeError]] = getattr(
_native, "ForkedAfterNativeRuntimeStarted", _NeverRaised
)
ProcessReservedForForking: Final[type[RuntimeError]] = getattr(_native, "ProcessReservedForForking", _NeverRaised)
def reserve_process_for_forking(where: str) -> None:
def reserve_process_for_forking(where: str, reserve: Callable[[], None] = _native.reserve_process_for_forking) -> None:
"""Forbid native routes in this process. Raises if one already ran here."""
native: Final = get_native_bridge()
reserve: Final = getattr(native, "reserve_process_for_forking", None)
if not callable(reserve):
return
try:
reserve()
except RuntimeError as error:

35
litellm/rust_bridge/loader.py
View file

@ -1,35 +0,0 @@
"""Loader for the packaged LiteLLM Rust extension."""
from __future__ import annotations
from types import ModuleType
from typing import Final
_BRIDGE_SENTINEL: Final = object()
_cached_bridge: ModuleType | None | object = _BRIDGE_SENTINEL
def get_native_bridge() -> ModuleType | None:
"""Return the packaged Rust extension, or ``None`` when unavailable."""
global _cached_bridge
if _cached_bridge is not _BRIDGE_SENTINEL:
return _cached_bridge if isinstance(_cached_bridge, ModuleType) else None
try:
from litellm.rust_bridge import _native
except ImportError:
_cached_bridge = None
return None
_cached_bridge = _native
return _native
def reset_native_bridge_cache() -> None:
"""Forget the cached extension so the next lookup reimports it from disk."""
global _cached_bridge
_cached_bridge = _BRIDGE_SENTINEL
def native_bridge_available() -> bool:
"""Whether the packaged Rust extension is importable."""
return get_native_bridge() is not None

25
litellm/rust_bridge/responses/websocket.py
View file

@ -2,32 +2,33 @@
from __future__ import annotations
from collections.abc import Awaitable
from dataclasses import dataclass
from typing import Final, Protocol
import httpx
from websockets.exceptions import ConnectionClosedOK
from litellm.rust_bridge import _native
from litellm.rust_bridge.host.timeouts import timeout_to_seconds
from litellm.rust_bridge.loader import get_native_bridge
class RustResponsesWebSocket(Protocol):
async def send_text(self, text: str) -> None: ...
def send_text(self, text: str) -> Awaitable[None]: ...
async def recv_text(self) -> str | None: ...
def recv_text(self) -> Awaitable[str | None]: ...
async def close(self) -> None: ...
def close(self) -> Awaitable[None]: ...
class RustResponsesWebSocketConnection(Protocol):
@classmethod
async def connect(
def connect(
cls,
url: str,
headers: dict[str, str],
timeout_seconds: float | None,
) -> RustResponsesWebSocket: ...
) -> Awaitable[RustResponsesWebSocket]: ...
class _Unset:
@ -53,16 +54,10 @@ def set_rust_responses_websocket(
_STATE.connection = connection
def load_rust_responses_websocket() -> RustResponsesWebSocketConnection | None:
def load_rust_responses_websocket() -> RustResponsesWebSocketConnection:
if _STATE.connection is not None:
return _STATE.connection
native_bridge: Final = get_native_bridge()
if native_bridge is None:
return None
connection_type: Final[RustResponsesWebSocketConnection | None] = getattr(
native_bridge, "ResponsesWebSocketConnection", None
)
return connection_type
return _native.ResponsesWebSocketConnection
class _ConnectionAdapter:
@ -89,8 +84,6 @@ async def connect(
timeout: float | httpx.Timeout | None,
) -> _ConnectionAdapter | None:
connection_type: Final = load_rust_responses_websocket()
if connection_type is None:
return None
try:
connection: Final = await connection_type.connect(
url=url,

19
litellm/rust_bridge/runtime.py
View file

@ -7,7 +7,8 @@ from typing import Final, Generic, NoReturn, TypeAlias, TypeVar
from typing_extensions import assert_never
from litellm.exceptions import APIError
from litellm.rust_bridge.bindings import NativeBinding, native_exception_types
from litellm.rust_bridge import _native
from litellm.rust_bridge.bindings import NativeBinding
from litellm.rust_bridge.catalog import RouteContext, Rules, decision
from litellm.rust_bridge.configuration import Decision
from litellm.rust_bridge.response_metadata import mark_rust_response
@ -153,15 +154,11 @@ def attempt(
) -> RustAttempt[ResultT]:
if native_call is None:
return RustUnavailable()
exceptions: Final = native_exception_types()
if exceptions is None:
return RustHandled(adapt(native_call()))
declined, upstream = exceptions
try:
value: Final = native_call()
except declined as error:
except _native.RustBridgeDeclined as error:
return RustDeclined(reason=_decline_reason(error))
except upstream as error:
except _native.RustUpstreamError as error:
_raise_upstream(error, context)
return RustHandled(adapt(value))
@ -174,15 +171,11 @@ async def aattempt(
) -> RustAttempt[ResultT]:
if native_call is None:
return RustUnavailable()
exceptions: Final = native_exception_types()
if exceptions is None:
return RustHandled(adapt(await native_call()))
declined, upstream = exceptions
try:
value: Final = await native_call()
except declined as error:
except _native.RustBridgeDeclined as error:
return RustDeclined(reason=_decline_reason(error))
except upstream as error:
except _native.RustUpstreamError as error:
_raise_upstream(error, context)
return RustHandled(adapt(value))

100
litellm/secret_managers/get_azure_ad_token_provider.py
View file

@ -1,14 +1,27 @@
import os
import time
from collections.abc import Callable
from dataclasses import dataclass
from types import TracebackType
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.single_flight import SingleFlight
from litellm.types.secret_managers.get_azure_ad_token_provider import (
AzureCredentialType,
)
_HELD_TOKEN_EXPIRY_MARGIN_SECONDS: Final = 60
if TYPE_CHECKING:
from azure.core.credentials import TokenProvider
from azure.core.credentials import (
AccessToken,
AccessTokenInfo,
SupportsTokenInfo,
TokenCredential,
TokenRequestOptions,
)
from typing_extensions import TypeIs # noqa: TID251 # tells the two azure-core credential protocols apart
def infer_credential_type_from_environment() -> AzureCredentialType:
@ -37,6 +50,86 @@ def infer_credential_type_from_environment() -> AzureCredentialType:
return AzureCredentialType.DefaultAzureCredential
@dataclass(slots=True)
class _HeldToken:
token: "AccessToken | AccessTokenInfo | None" = None
def still_valid(self) -> str | None:
held: Final = self.token
if held is None or held.expires_on - time.time() <= _HELD_TOKEN_EXPIRY_MARGIN_SECONDS:
return None
return held.token
@dataclass(slots=True)
class _TokenInfoWatcher:
credential: "SupportsTokenInfo"
held: _HeldToken
def get_token_info(self, *scopes: str, options: "TokenRequestOptions | None" = None) -> "AccessTokenInfo":
token: Final = self.credential.get_token_info(*scopes, options=options)
self.held.token = token
return token
def close(self) -> None:
self.credential.close()
def __enter__(self) -> "SupportsTokenInfo":
return self.credential.__enter__()
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
self.credential.__exit__(exc_type, exc_value, traceback)
@dataclass(slots=True)
class _TokenWatcher:
credential: "TokenCredential"
held: _HeldToken
def get_token(
self,
*scopes: str,
claims: str | None = None,
tenant_id: str | None = None,
enable_cae: bool = False,
**kwargs: object, # kwargs-ok: azure-core forwards its request options verbatim to the wrapped credential
) -> "AccessToken":
token: Final = self.credential.get_token(
*scopes, claims=claims, tenant_id=tenant_id, enable_cae=enable_cae, **kwargs
)
self.held.token = token
return token
def _supports_token_info(
credential: "TokenCredential | SupportsTokenInfo",
) -> "TypeIs[SupportsTokenInfo]":
return hasattr(credential, "get_token_info")
def single_flight_bearer_token_provider(
credential: "TokenCredential | SupportsTokenInfo", scope: str
) -> Callable[[], str]:
from azure.identity import get_bearer_token_provider
held: Final = _HeldToken()
watched: Final = (
_TokenInfoWatcher(credential, held) if _supports_token_info(credential) else _TokenWatcher(credential, held)
)
provider: Final = get_bearer_token_provider(watched, scope)
token_request: Final[SingleFlight[str]] = SingleFlight()
def bearer_token() -> str:
return token_request.run(provider, while_busy=held.still_valid())
return bearer_token
def get_azure_ad_token_provider(
azure_scope: str | None = None,
azure_credential: AzureCredentialType | None = None,
@ -65,7 +158,6 @@ def get_azure_ad_token_provider(
DefaultAzureCredential,
ManagedIdentityCredential,
WorkloadIdentityCredential,
get_bearer_token_provider,
)
if azure_scope is None:
@ -77,7 +169,7 @@ def get_azure_ad_token_provider(
else None or os.environ.get("AZURE_CREDENTIAL") or infer_credential_type_from_environment()
)
verbose_logger.info("For Azure AD Token Provider, choosing credential type: %s", cred)
credential: TokenProvider | None = None
credential: TokenCredential | SupportsTokenInfo | None = None
if cred == AzureCredentialType.ClientSecretCredential:
credential = ClientSecretCredential(
client_id=os.environ["AZURE_CLIENT_ID"],
@ -133,4 +225,4 @@ def get_azure_ad_token_provider(
if credential is None:
raise ValueError("No credential provided")
return get_bearer_token_provider(credential, azure_scope)
return single_flight_bearer_token_provider(credential, azure_scope)

4
litellm/secret_managers/main.py
View file

@ -5,7 +5,6 @@ import time
import traceback
from collections.abc import Callable
from dataclasses import dataclass
from functools import lru_cache
from typing import Final
import httpx
@ -14,6 +13,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
from litellm.litellm_core_utils.single_flight import single_flight_lru_cache
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.secret_managers.get_azure_ad_token_provider import (
get_azure_ad_token_provider,
@ -27,7 +27,7 @@ _PARSED_LITERAL: Final = TypeAdapter(object)
_SECRET_MANAGER_CLIENT: Final = TypeAdapter(object)
@lru_cache(maxsize=128)
@single_flight_lru_cache
def _cached_azure_oidc_token_provider(audience: str) -> Callable[[], str]:
return get_azure_ad_token_provider(azure_scope=audience)

18
litellm/tracing/otlp_http.py
View file

@ -1,20 +1,9 @@
import json
from typing import Final, Protocol, runtime_checkable
from typing import Final
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm.rust_bridge.loader import get_native_bridge
@runtime_checkable
class NativeOtlpError(Protocol):
def trace_encode_error(self, message: str) -> bytes: ...
_NATIVE: Final[TypeAdapter[NativeOtlpError]] = TypeAdapter(
NativeOtlpError, config=ConfigDict(arbitrary_types_allowed=True)
)
from litellm.rust_bridge import _native
class OTLPError(TypedDict):
@ -22,8 +11,7 @@ class OTLPError(TypedDict):
def encode_error(message: str) -> bytes:
native: Final = get_native_bridge()
return b"" if native is None else _NATIVE.validate_python(native).trace_encode_error(message)
return _native.trace_encode_error(message)
def encode_otlp_response(content_type: str | None, error: str | None = None) -> tuple[bytes, str]:

14
litellm/types/agents.py
View file

@ -3,7 +3,7 @@ from datetime import datetime
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, TypeAlias
from urllib.parse import urlsplit
from pydantic import ConfigDict, Field, PrivateAttr, StrictInt, field_validator
from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, StrictInt, field_validator
from typing_extensions import ReadOnly, Required, TypedDict
from litellm.types.llms.base import LiteLLMBaseModel, LiteLLMPydanticObjectBase
@ -343,6 +343,18 @@ class ListAgentsResponse(LiteLLMBaseModel):
agents: list[AgentResponse]
class AgentCreateRequest(BaseModel):
model_config = ConfigDict(frozen=True, extra="forbid")
id: str
base_agent: str | None = None
agent_config: Mapping[str, object] | None = None
system_instruction: str | None = None
base_environment: str | Mapping[str, object] | None = None
description: str | None = None
tools: Sequence[Mapping[str, object]] | None = None
class AgentCreateResponse(LiteLLMPydanticObjectBase):
"""
Response from a provider-side agent creation or get call (e.g. Gemini v1beta/agents).

3
litellm/types/utils.py
View file

@ -225,6 +225,8 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_xhigh_reasoning_effort: bool | None
supports_max_reasoning_effort: bool | None
reasoning_effort_levels: ReadOnly[Sequence[str] | None]
reasoning_modes: ReadOnly[Sequence[str] | None]
reasoning_contexts: ReadOnly[Sequence[str] | None]
default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None]
supports_output_config: bool | None
supports_image_size: bool | None
@ -4587,6 +4589,7 @@ class LlmProviders(str, Enum):
PRISM = "prism"
OPENINFER = "openinfer"
INFERENCEAPIS = "inferenceapis"
ZEROGPU = "zerogpu"
DARKBLOOM = "darkbloom"
META = "meta"
SAIL = "sail"

9
litellm/utils.py
View file

@ -7001,6 +7001,8 @@ def get_model_info_helper(
supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None),
supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None),
reasoning_effort_levels=_model_info.get("reasoning_effort_levels", None),
reasoning_modes=_model_info.get("reasoning_modes", None),
reasoning_contexts=_model_info.get("reasoning_contexts", None),
default_reasoning_effort=_model_info.get("default_reasoning_effort", None),
bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None),
bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None),
@ -9620,18 +9622,19 @@ class ProviderConfigManager:
return litellm.HostedVLLMDecisionsConfig()
if provider == LlmProviders.LM_STUDIO:
return litellm.LMStudioDecisionsConfig()
if provider in (LlmProviders.OLLAMA, LlmProviders.OLLAMA_CHAT):
return litellm.OllamaDecisionsConfig()
if provider == LlmProviders.LAYA:
return litellm.LayaDecisionsConfig()
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE):
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI):
configs: Final = {
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
LlmProviders.AZURE: litellm.AzureDecisionsConfig,
LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig,
}
return configs[provider]()
if provider == LlmProviders.AZURE_AI:
return litellm.AzureAIDecisionsConfig()
return None
@staticmethod

1527
model_prices_and_context_window.json
View file

File diff suppressed because it is too large Load diff

28
provider_endpoints_support.json
View file

@ -1225,7 +1225,7 @@
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_generations": true,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
@ -4437,6 +4437,32 @@
"generateContent": true
}
},
"zerogpu": {
"display_name": "ZeroGPU (`zerogpu`)",
"url": "https://docs.litellm.ai/docs/providers/zerogpu",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"ragflow": {
"display_name": "RAGFlow (`ragflow`)",
"url": "https://docs.litellm.ai/docs/providers/ragflow",

2
pyproject.toml
View file

@ -401,7 +401,7 @@ markers = [
"asyncio: mark test as an asyncio test",
"limit_leaks: mark test with memory limit for leak detection (e.g., '40 MB')",
"no_parallel: mark test to run sequentially (not in parallel) - typically for memory measurement tests",
"requires_rust_extension: public Python contract requiring an enabled, compiled Rust extension",
"requires_rust_extension: public Python contract tests that run against the release-built wheel",
]
filterwarnings = [
# Suppress Pydantic serializer warnings from mock server responses (non-critical for memory tests)

71
schema.prisma
View file

@ -1163,6 +1163,7 @@ model LiteLLM_ManagedFileTable {
@@index([unified_file_id])
@@index([flat_model_file_ids], type: Gin)
@@index([team_id, created_at(sort: Desc)])
@@index([created_by, created_at(sort: Desc)])
}
model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the
@ -2060,3 +2061,73 @@ model LiteLLM_TelemetryReport {
@@index([window_end])
@@index([created_at])
}
enum AccountingEventKind {
inbound_usage
outbound_usage
cost
}
enum AccountingDeliveryState {
pending
applied
parked
}
enum AccountingDeliveryTarget {
LiteLLM_VerificationToken
LiteLLM_UserTable
LiteLLM_TeamTable
LiteLLM_TeamMembership
LiteLLM_OrganizationTable
LiteLLM_OrganizationMembership
LiteLLM_EndUserTable
LiteLLM_AgentsTable
LiteLLM_TagTable
LiteLLM_ProjectTable
LiteLLM_ModelAccessGroupBudgetTable
LiteLLM_BudgetTable
LiteLLM_BudgetWindowSpend
LiteLLM_DailyUserSpend
LiteLLM_DailyTeamSpend
LiteLLM_DailyOrganizationSpend
LiteLLM_DailyEndUserSpend
LiteLLM_DailyAgentSpend
LiteLLM_DailyTagSpend
LiteLLM_DailyModelUsage
LiteLLM_DailyToolSpend
LiteLLM_AutoRouterDailySpend
}
// Durable accounting ledger. One row per usage or cost event; payload holds the
// private event envelope versioned by schema_version. Usage rows retain request-scoped
// pricing context; cost rows include resolved rates and the finalized pricing snapshot.
model LiteLLM_AccountingEvent {
event_id String @id
kind AccountingEventKind
source_event_id String?
schema_version Int
payload_hash String
occurred_at DateTime
recorded_at DateTime @default(now())
payload Json
@@index([recorded_at])
@@index([kind, source_event_id])
}
// One row per (event, target table, version): pending work for the projector and,
// once applied in the same transaction as the target write, the receipt that stops a retry.
model LiteLLM_AccountingDelivery {
event_id String
target AccountingDeliveryTarget
version Int
state AccountingDeliveryState @default(pending)
attempts Int @default(0)
next_attempt_at DateTime @default(now())
applied_at DateTime?
last_error String?
@@id([event_id, target, version])
@@index([target, state, next_attempt_at])
}

20
tests/base_sdk_tests/check_base_sdk_install.py
View file

@ -8,6 +8,8 @@ the very class of undeclared-dependency bug this guards against.
import argparse
import importlib.util
from importlib.machinery import EXTENSION_SUFFIXES
from pathlib import Path
import re
import sys
import traceback
@ -89,6 +91,18 @@ def check_import() -> str:
return f"imported {distribution} {sdk_version}"
def check_native_extension() -> str:
import litellm
from litellm.rust_bridge import _native
native_path: Final = Path(_native.__file__).resolve()
package_path: Final = Path(litellm.__file__).resolve().parent
_require(native_path.parent == package_path / "rust_bridge", "native extension is outside the SDK package")
_require(any(str(native_path).endswith(suffix) for suffix in EXTENSION_SUFFIXES), "native module is not compiled")
_require(_native.trace_encode_error("ping") == b"\x12\x04ping", "native OTLP error encoding failed")
return f"loaded and exercised {native_path}"
def check_completion() -> str:
import litellm
@ -230,6 +244,7 @@ def check_bedrock_credential_resolution() -> str:
CHECKS: tuple[tuple[str, Callable[[], str]], ...] = (
("environment is base-only", check_environment_is_base_only),
("import litellm", check_import),
("native extension", check_native_extension),
("optional MCP installation guidance", check_mcp_install_guidance),
("chat completion", check_completion),
("chat completion with a function tool", check_completion_with_function_tool),
@ -250,8 +265,11 @@ def _run(check: Callable[[], str]) -> tuple[bool, str]:
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--profile", choices=("legacy", "core", "dependencies"), default="legacy")
parser.add_argument("--profile", choices=("legacy", "core", "dependencies", "native"), default="legacy")
profile = parser.parse_args().profile
if profile == "native":
print(check_native_extension())
return 0
for module in ("boto3", "botocore", "tokenizers", "huggingface_hub"):
present = importlib.util.find_spec(module) is not None
_require(present == (profile != "core"), f"{profile}: unexpected presence of {module}: {present}")

5
tests/code_coverage_tests/router_code_coverage.py
View file

@ -95,6 +95,7 @@ ignored_function_names = [
"arm_routing_read_prefetch", # Tested in tests/unit/caching/test_request_redis_batch_pre_call.py (file lacks "router" in name)
"_deployment_params_base_model", # Tested through log_retry in test_router_helper_utils.py
"_configured_model_info", # Tested through get_configured_service_tiers in test_router.py
"_every_deployment_refused", # Tested through the unsupported transcription retry tests in test_router.py
"_routable_deployments", # Tested through get_configured_service_tiers and get_routable_upstream_model in test_router.py
"_async_get_available_deployment", # Body of the `route {model}` phase wrapper, exercised through async_get_available_deployment in test_router.py
"_async_get_available_deployment_for_pass_through", # Same, through async_get_available_deployment_for_pass_through in test_router.py
@ -174,6 +175,10 @@ ignored_function_names = [
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
"_responses_event_announces_background_run", # Tested through the background Responses stream tests in test_router.py
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
]

15
tests/integration/compatibility/test_missing_body_param_status.py
View file

@ -254,8 +254,13 @@ _ROUTES: Final[dict[str, _Route]] = {
"acreate_agent": (
"/v1beta/agents",
"/v1beta/agents",
("name",),
{"name": "agent", "base_agent": "waverunner", "instructions": "You are a helpful assistant."},
("id",),
{
"id": "agent",
"base_agent": "antigravity-preview-09-2026",
"system_instruction": "You are a helpful assistant.",
"agent_config": {"type": "antigravity", "model": "gemini-3.8-flash"},
},
"gemini/gemini-2.5-flash",
),
"acreate_interaction": (
@ -1250,11 +1255,7 @@ def test_valid_agent_creation_reaches_upstream(gateway: Gateway) -> None:
_assert_scripted_response("acreate_agent", caller)
matches: Final = _observed(_Observations(gateway.upstream_url), identity)
outbound: Final = object_value(matches[0]["body"])
assert outbound == {
"name": "agent",
"base_agent": "waverunner",
"instructions": "You are a helpful assistant.",
}, matches
assert outbound == body, matches
def test_valid_speech_returns_binary_audio_and_reaches_upstream(gateway: Gateway) -> None:

128
tests/integration/database/test_accounting_ledger.py Normal file
View file

@ -0,0 +1,128 @@
import os
import subprocess
import sys
from collections.abc import Iterator
from pathlib import Path
from typing import Final
import psycopg
import pytest
from psycopg import sql
from litellm.accounting import DeliveryState, DeliveryTarget, EventKind
from tests.integration._support.database import scratch_database
_SCHEMA: Final = Path(__file__).resolve().parents[3] / "litellm-proxy-extras/litellm_proxy_extras/schema.prisma"
_EVENT_INSERT: Final = (
'INSERT INTO "LiteLLM_AccountingEvent" '
"(event_id, kind, schema_version, payload_hash, occurred_at, payload) "
"VALUES ('event', %s, %s, 'hash', '2026-01-01', '{}') RETURNING kind::text, schema_version"
)
_DELIVERY_INSERT: Final = (
'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version, state, attempts) '
"VALUES ('event', %s, %s, %s, %s) RETURNING target::text, version, state::text, attempts"
)
@pytest.fixture(scope="module")
def ledger_database() -> Iterator[str]:
with scratch_database("accounting") as database_url:
result: Final = subprocess.run(
[sys.executable, "-I", "-m", "prisma", "migrate", "deploy", "--schema", str(_SCHEMA)],
env={**os.environ, "DATABASE_URL": database_url},
capture_output=True,
text=True,
timeout=300,
)
assert result.returncode == 0, result.stdout + result.stderr
yield database_url
@pytest.fixture
def ledger(ledger_database: str) -> Iterator[psycopg.Connection[tuple[object, ...]]]:
with psycopg.connect(ledger_database) as connection:
try:
yield connection
finally:
connection.rollback()
@pytest.mark.parametrize("kind", tuple(EventKind))
def test_ledger_accepts_every_public_event_kind(
ledger: psycopg.Connection[tuple[object, ...]], kind: EventKind
) -> None:
row: Final = ledger.execute(_EVENT_INSERT, (kind.value, 1)).fetchone()
assert row == (kind.value, 1)
@pytest.mark.parametrize("target", tuple(DeliveryTarget))
@pytest.mark.parametrize("state", tuple(DeliveryState))
def test_delivery_accepts_every_public_target_and_state(
ledger: psycopg.Connection[tuple[object, ...]], target: DeliveryTarget, state: DeliveryState
) -> None:
row: Final = ledger.execute(_DELIVERY_INSERT, (target.value, 1, state.value, 0)).fetchone()
assert row == (target.value, 1, state.value, 0)
@pytest.mark.parametrize(
("enum_name", "values"),
[
("AccountingEventKind", tuple(value.value for value in EventKind)),
("AccountingDeliveryState", tuple(value.value for value in DeliveryState)),
("AccountingDeliveryTarget", tuple(value.value for value in DeliveryTarget)),
],
)
def test_migrated_enums_match_the_public_contract(
ledger: psycopg.Connection[tuple[object, ...]], enum_name: str, values: tuple[str, ...]
) -> None:
rows: Final = ledger.execute(
"SELECT e.enumlabel FROM pg_enum e JOIN pg_type t ON t.oid = e.enumtypid WHERE t.typname = %s",
(enum_name,),
).fetchall()
assert frozenset(row[0] for row in rows) == frozenset(values)
@pytest.mark.parametrize("version", [0, -1])
def test_event_rejects_nonpositive_schema_versions(
ledger: psycopg.Connection[tuple[object, ...]], version: int
) -> None:
with pytest.raises(psycopg.errors.CheckViolation, match="AccountingEvent_schema_version_check"):
ledger.execute(_EVENT_INSERT, (EventKind.OUTBOUND_USAGE.value, version))
@pytest.mark.parametrize(
("version", "attempts", "constraint"),
[
(0, 0, "AccountingDelivery_version_check"),
(-1, 0, "AccountingDelivery_version_check"),
(1, -1, "AccountingDelivery_attempts_check"),
],
)
def test_delivery_rejects_invalid_versions_and_attempts(
ledger: psycopg.Connection[tuple[object, ...]], version: int, attempts: int, constraint: str
) -> None:
with pytest.raises(psycopg.errors.CheckViolation, match=constraint):
ledger.execute(_DELIVERY_INSERT, (DeliveryTarget.KEY_SPEND.value, version, "pending", attempts))
@pytest.mark.parametrize("column", ["state", "target"])
def test_delivery_rejects_unknown_enum_values(ledger: psycopg.Connection[tuple[object, ...]], column: str) -> None:
ledger.execute(_DELIVERY_INSERT, (DeliveryTarget.KEY_SPEND.value, 1, "pending", 0))
with pytest.raises(psycopg.errors.InvalidTextRepresentation, match="invalid input value for enum"):
ledger.execute(
sql.SQL('UPDATE "LiteLLM_AccountingDelivery" SET {} = %s').format(sql.Identifier(column)), ("invalid",)
)
def test_event_rejects_unknown_kind(ledger: psycopg.Connection[tuple[object, ...]]) -> None:
with pytest.raises(psycopg.errors.InvalidTextRepresentation, match="invalid input value for enum"):
ledger.execute(_EVENT_INSERT, ("invalid", 1))
def test_new_delivery_defaults_to_pending_with_no_attempts(ledger: psycopg.Connection[tuple[object, ...]]) -> None:
row: Final = ledger.execute(
'INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version) '
"VALUES ('event', %s, 1) RETURNING state::text, attempts",
(DeliveryTarget.KEY_SPEND.value,),
).fetchone()
assert row == ("pending", 0)

Some files were not shown because too many files have changed in this diff Show more