Merge remote-tracking branch 'origin/main' into litellm_responses_calltypes

# Conflicts:
#	ui/litellm-dashboard/src/lib/http/schema.d.ts
This commit is contained in:
mateo-berri 2026-10-31 13:53:00 -07:00
commit 9da5899763
209 changed files with 36500 additions and 18482 deletions

7
.circleci/AGENTS.md Normal file
View file

@ -0,0 +1,7 @@
# .circleci
The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base
A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base

View file

@ -21,19 +21,30 @@ orbs:
win: circleci/windows@5.0 # Add Windows orb
commands:
checkout_migration_source:
checkout_over_https:
parameters:
revision:
type: string
default: ""
shell:
type: string
default: /bin/bash -eo pipefail
steps:
- run:
name: Select the requested migration test revision
name: Check out the public repository over HTTPS
shell: << parameters.shell >>
environment:
MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
REQUESTED_REVISION: << parameters.revision >>
command: |
revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
git init
git remote add origin https://github.com/BerriAI/litellm.git
git fetch --depth 1 origin "$revision"
git checkout --detach FETCH_HEAD
for attempt in 1 2 3; do
git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
[[ $attempt -lt 3 ]] || exit 1
sleep 5
done
skip_if_unrelated_changes:
parameters:
category:
@ -347,7 +358,7 @@ commands:
- when:
condition: << parameters.checkout >>
steps:
- checkout
- checkout_over_https
- setup_google_dns
- install_uv
- install_rust
@ -377,7 +388,8 @@ jobs:
CARGO_HTTP_MULTIPLEXING: "false"
CARGO_NET_RETRY: "5"
steps:
- checkout
- checkout_over_https:
shell: bash.exe
- run:
name: Skip job when no windows-release-relevant files changed
shell: bash.exe
@ -451,7 +463,7 @@ jobs:
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- setup_google_dns
- install_uv
- install_rust
@ -482,7 +494,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
@ -559,7 +571,7 @@ jobs:
resource_class: medium
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -659,7 +671,7 @@ jobs:
resource_class: large
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -740,7 +752,7 @@ jobs:
resource_class: large
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes:
category: client
- setup_google_dns
@ -794,7 +806,7 @@ jobs:
# the rest of the suite: nothing else performs a logout.
PROXY_LOGOUT_URL: "https://www.example.com"
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -940,7 +952,7 @@ jobs:
# and by migration.serverRootPath.config.ts, which refuses to run without it.
SERVER_ROOT_PATH: "/litellm"
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -1066,11 +1078,12 @@ jobs:
- when:
condition: << parameters.migration_qualification >>
steps:
- checkout_migration_source
- checkout_over_https:
revision: << pipeline.parameters.migration_source_sha >>
- unless:
condition: << parameters.migration_qualification >>
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- run:
@ -1122,7 +1135,8 @@ jobs:
MIGRATION_TEST_OUTPUT: /tmp/migration-results
PYTHONPATH: tests/e2e
steps:
- checkout_migration_source
- checkout_over_https:
revision: << pipeline.parameters.migration_source_sha >>
- run:
name: Generate LiteLLM master key
command: |
@ -1197,7 +1211,7 @@ jobs:
resource_class: medium
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
@ -1371,7 +1385,7 @@ jobs:
environment:
UV_PYTHON_DOWNLOADS: never
steps:
- checkout
- checkout_over_https
- install_uv
- run:
name: Install the locked shard verifier and coverage dependencies
@ -1419,7 +1433,7 @@ jobs:
ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
command: |
[[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
git rm -r -f --quiet litellm enterprise litellm-proxy-extras
git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
git reset --quiet
@ -1494,7 +1508,7 @@ jobs:
environment:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- setup_litellm_test_deps:
checkout: false

View file

@ -4,9 +4,14 @@
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
"## Affected release" with nothing under it must not appear in the final description -->
<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
## Handwritten 1-3 sentence description
<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
plain sentences proves the author understands what it does, and human-written technical English is far easier
for a reviewer to parse than AI-written English
An AI agent may draft a starting point here, but the author replaces it with their own handwritten version
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
No bullets, tables, or code names here. Everything else goes inside the Details block below -->
<details>
<summary>Details</summary>

View file

@ -2,7 +2,7 @@ name: Create RC Branch
on:
schedule:
- cron: "0 3 * * 5"
- cron: "0 3 * * 1"
timezone: "America/Los_Angeles"
workflow_dispatch:

View file

@ -200,6 +200,7 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
unpriced: Final = _without_read_price(billed.response)
cost: Final = logging_obj.response_cost_calculator(result=unpriced)
end_time: Final = datetime.now()
logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
async_coroutine=logging_obj.async_success_handler(
result=unpriced, start_time=logging_obj.start_time, end_time=end_time

View file

@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
("max_tokens", "length"),
("refusal", "content_filter"),
("compaction", "length"),
("pause_turn", "length"),
("model_context_window_exceeded", "length"),
("guardrail_intervened", "content_filter"),
("content_filtered", "content_filter"),
@ -75,6 +76,7 @@ mod tests {
#[case::end_turn("end_turn", "stop")]
#[case::stop_sequence("stop_sequence", "stop")]
#[case::max_tokens("max_tokens", "length")]
#[case::pause_turn("pause_turn", "length")]
#[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
#[case::refusal("refusal", "content_filter")]
#[case::guardrail_intervened("guardrail_intervened", "content_filter")]

View file

@ -0,0 +1 @@
The price vocabulary (meter, band, channel, `When`, `Billing`) is explained in [../model-catalog/AGENTS.md](../model-catalog/AGENTS.md#concepts), the only crate that maps it to JSON keys

View file

@ -19,6 +19,7 @@ pub enum ServiceTier {
Balanced,
}
/// A prompt-size range. `OverPromptTokens(n)` is spelled `_above_<n>_tokens` or `_above_<n / 1000>k_tokens`
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Band {
#[default]
@ -26,12 +27,15 @@ pub enum Band {
OverPromptTokens(NonZeroU64),
}
/// The request condition a price applies under, its service tier and prompt band
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct When {
pub tier: ServiceTier,
pub band: Band,
}
/// The route whose price list a call reads, from the `_batches` and `_decisions` suffixes. Not part of `When`,
/// since a missing batch price is half the online rate and a missing decisions price is the full online rate
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Channel {
#[default]
@ -40,6 +44,7 @@ pub enum Channel {
Decisions,
}
/// The key of one rate, what is counted, through which channel and under which condition
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Price {
pub meter: Meter,
@ -47,6 +52,7 @@ pub struct Price {
pub when: When,
}
/// Every rate a row lists, its off-peak overlay and the row attributes that say how the rates apply
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Prices {
rates: BTreeMap<Price, UnitPrice>,
@ -117,6 +123,7 @@ pub enum TierScope {
WholeCall,
}
/// Rates that replace the row's token rates while a UTC window in the schedule is open
#[derive(Clone, Debug, PartialEq)]
pub struct OffPeak {
schedule: Schedule,
@ -133,6 +140,7 @@ pub struct OffPeakRates {
pub reasoning: Option<UnitPrice>,
}
/// Multipliers applied on top of a priced call, never prices themselves
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Adjustments(BTreeMap<Uplift, Multiplier>);

View file

@ -1,5 +1,6 @@
use std::num::NonZeroU64;
/// What a price counts. A variation only one meter has, such as cache TTL or image size, lives inside its variant
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Meter {
Input(InputUnit),
@ -51,12 +52,14 @@ pub enum OutputUnit {
VideoSecond,
}
/// `Explicit` is a cache the caller created ahead of time, spelled `explicit_cache_read_*`. `Implicit` is every other read
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum CacheReadMode {
Implicit,
Explicit,
}
/// How long a cache write lives. `OneHour` and `TwelveHours` are spelled `_above_1hr` and `_above_12hr`
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum CacheTtl {
Default,

View file

@ -1,3 +1,41 @@
## Concepts
Every price in a row lowers to one `Price { meter, channel, when }` and its `UnitPrice`. These keys come from the `FlatKey::parse` tests
- `input_cost_per_token`
- meter `Input(Token(Text))`, channel `Online`, tier `Standard`, band `Base`
- `output_cost_per_token_above_272k_tokens_priority`
- meter `Output(Token(Text))`, channel `Online`
- `_priority` sets tier `Priority`
- `_above_272k_tokens` sets band `OverPromptTokens(272000)`
- `cache_creation_input_token_cost_above_1hr_above_200k_tokens`
- meter `CacheWrite { Text, OneHour }`, where `_above_1hr` is the cache TTL, not a band
- `_above_200k_tokens` sets band `OverPromptTokens(200000)`
- channel `Online`, tier `Standard`
- `ocr_cost_per_page_batches`
- meter `Feature(OcrPage)`, tier `Standard`, band `Base`
- `_batches` sets channel `Batch`
- Meter is what gets counted, such as text input tokens, reasoning tokens, an image of a given size or a web search
- A variation only one meter has stays inside it (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`)
- `UnitPrice` is USD for one unit of the meter, such as one token, image, page or second
- Band is a range of prompt sizes, starting where `_above_<N>_tokens` or `_above_<N>k_tokens` says
- `Prices::band` picks one band from prompt tokens alone, the highest edge passed wins, and `Band::Base` is below every edge
- The whole call moves, so every token meter, output included, bills all its tokens at that band's rate
- `Billing::band_edge` says whether a prompt exactly at the edge is in the band (exclusive by default, inclusive for xAI)
- A band only another tier or the batch channel prices never moves the call
- `When` is the request's condition, its service tier and band
- A missing price falls back from `(tier, band)` to `(standard, band)`, `(tier, base)`, then `(standard, base)`
- `Channel` is the route whose price list the call reads, and each falls back differently
- `Online` falls back across `When` as above
- `Batch` (`_batches`) uses bands but no service tier, and a missing price is half the online rate
- `Decisions` (`_decisions`, OpenAI `/v1/decisions`) overrides single online fields, and a missing price is the full online rate
- Off-peak and `Adjustments` are never prices
- Off-peak swaps in other token rates while a UTC window is open
- `Adjustments` multiplies a priced call for region, residency, geo or `speed: fast`
- `Billing` on `Prices` holds row attributes that change how its prices apply (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope)
- `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
## Layout
`wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read
@ -6,8 +44,6 @@
`catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations`
`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
`store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md)
`layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller
@ -16,7 +52,7 @@ Catalog errors use one opaque `Error` with typed causes and JSON Pointer accesso
## Prices
The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices
The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names
A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs`

View file

@ -21,6 +21,7 @@ _T = TypeVar("_T")
# When True, suppresses async logging and billing for internal sub-calls
# (e.g., emulated file-search steps that make nested LLM calls).
is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
internal_parent_request_id: Final[ContextVar[str | None]] = ContextVar("internal_parent_request_id", default=None)
# One request prices its totals, its per-token-type lines and the rates it reports on
# separate code paths. Each reads the clock for off-peak pricing, so without a pinned
@ -40,13 +41,15 @@ _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=Fa
@contextmanager
def internal_sub_call() -> Generator[None]:
def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
"""A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
token: Final = is_internal_call.set(True)
parent_token: Final = internal_parent_request_id.set(parent_request_id or internal_parent_request_id.get())
try:
yield
finally:
is_internal_call.reset(token)
internal_parent_request_id.reset(parent_token)
@contextmanager

View file

@ -3,7 +3,7 @@
The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
These inputs, including the effective cost-map rates, must survive request teardown.
Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
"""
@ -38,8 +38,8 @@ class CostCompiler(Protocol):
async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
"""Reuse the committed charge or finalize one charge and its delivery intent.
Resolve captured request-scoped context by event identity and cost-map rates
at compilation. Persist the resolved inputs in the required snapshot. Replay
Resolve captured request-scoped context and rates by event identity.
Persist the resolved inputs in the required snapshot. Replay
reuses the committed result and never consumes free allowance again.
Missing pricing is Unpriced, never an inferred zero.
"""

View file

@ -276,8 +276,8 @@ class NamedCost(BaseModel):
class PricingSnapshot(BaseModel):
"""Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
Retries reuse the committed charge and snapshot.
Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
Component sums and float tolerances belong to reconciliation, not this structural contract.
"""

View file

@ -3,9 +3,10 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from dataclasses import replace as dataclasses_replace
from enum import Enum
from itertools import groupby
from typing import Any, Final, Literal, cast
from pydantic import TypeAdapter
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -15,6 +16,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
from litellm.llms.vertex_ai.batches.transformation import (
is_native_vertex_batch_output_row,
native_vertex_batch_row_model,
native_vertex_batch_row_stats,
)
from litellm.types.llms.openai import Batch
@ -210,6 +212,7 @@ class _BatchOutputLineStats:
cache_creation_tokens: int
reasoning_tokens: int
model: str | None
usage: Usage | None
def _classify_output_line_stats(
@ -268,7 +271,7 @@ def _compute_output_line_stats(
raw_model: Final = response_body.get("model")
response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
completion_details: Final = usage.completion_tokens_details
line_prompt_cost, line_completion_cost = _output_line_cost(
line_prompt_cost, line_completion_cost = _output_line_cost_or_pending(
response_body=response_body,
usage=usage,
custom_llm_provider=custom_llm_provider,
@ -286,7 +289,15 @@ def _compute_output_line_stats(
cache_read_tokens=prompt_details["cache_hit_tokens"],
cache_creation_tokens=prompt_details["cache_creation_tokens"],
reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
model=response_model,
model=response_model or model_name,
usage=(
usage.model_copy(update=ocr_usage.model_dump(exclude_none=True))
if (ocr_usage := _ocr_usage_info_from_response_body(response_body)) is not None
and ocr_usage.model_dump(exclude_none=True)
else usage
if response_body.get("usage") is not None or usage.total_tokens > 0
else None
),
)
@ -298,6 +309,54 @@ def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> O
return OCRUsageInfo.model_validate(raw_usage_info)
def _output_line_cost_or_pending(
response_body: Mapping[str, object],
usage: Usage,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None,
response_model: str | None,
model_info: ModelInfo | None,
vertex_location: str | None,
) -> tuple[float, float]:
try:
return _output_line_cost(
response_body, usage, custom_llm_provider, model_name, response_model, model_info, vertex_location
)
except Exception: # noqa: BLE001 # retain measured usage when existing batch pricing fails
verbose_logger.warning("Batch line pricing unavailable; retaining usage with the existing zero-cost fallback")
return 0.0, 0.0
def _combined_reported_batch_usage(usages: tuple[Usage | None, ...]) -> Mapping[str, object] | None:
from litellm.cost_calculator import BaseTokenUsageProcessor
reported: Final = [usage for usage in usages if usage is not None]
if not reported:
return None
fields: Final = tuple(TypeAdapter(dict[str, JsonValue]).validate_python(usage.model_dump()) for usage in reported)
quantities: Final = {
key: sum(value for usage in fields if isinstance(value := usage.get(key), (int, float)))
for key in OCRUsageInfo.model_fields
if any(isinstance(usage.get(key), (int, float)) for usage in fields)
}
return {**BaseTokenUsageProcessor.combine_usage_objects(usage_objects=reported).model_dump(), **quantities}
def _batch_usage_by_request(usages: tuple[tuple[str | None, Usage | None], ...]) -> tuple[Mapping[str, object], ...]:
rows: Final = tuple(
{"model": model, "usage": usage.model_dump() if usage is not None else None} for model, usage in usages
)
return tuple(sorted(rows, key=lambda row: json.dumps(row, sort_keys=True)))
def _batch_usage_by_model(usages: tuple[tuple[str | None, Usage | None], ...]) -> Mapping[str, object]:
ordered: Final = sorted(usages, key=lambda item: item[0] or "")
return {
model: _combined_reported_batch_usage(tuple(item[1] for item in group))
for model, group in groupby(ordered, key=lambda item: item[0] or "")
}
def _output_line_cost(
response_body: Mapping[str, object],
usage: Usage,
@ -361,6 +420,9 @@ def _aggregate_batch_cost_usage_models(
completion_tokens=sum(stats.completion_tokens for stats in line_stats),
reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
**cache_token_params,
batch_usage_by_model=_batch_usage_by_model(tuple((stats.model, stats.usage) for stats in line_stats)),
batch_usage_by_request=_batch_usage_by_request(tuple((stats.model, stats.usage) for stats in line_stats)),
batch_unknown_usage_requests=sum(stats.usage is None for stats in line_stats),
)
batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0)
@ -401,17 +463,21 @@ def calculate_vertex_ai_batch_cost_and_usage(
from litellm.cost_calculator import batch_cost_calculator
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
row_stats: Final = tuple(
native_vertex_batch_row_stats(
row,
model_name,
model_info=model_info,
calculate_usage=VertexGeminiConfig.calculate_usage,
cost_calculator=batch_cost_calculator,
vertex_location=vertex_location,
observations: Final = tuple(
(
native_vertex_batch_row_model(row, model_name),
native_vertex_batch_row_stats(
row,
model_name,
model_info=model_info,
calculate_usage=VertexGeminiConfig.calculate_usage,
cost_calculator=batch_cost_calculator,
vertex_location=vertex_location,
),
)
for row in vertex_ai_batch_responses
)
row_stats: Final = tuple(stats for _, stats in observations)
priced: Final = tuple(stats for stats in row_stats if stats is not None)
total_prompt_cost: Final = sum(stats.prompt_cost for stats in priced)
total_completion_cost: Final = sum(stats.completion_cost for stats in priced)
@ -435,6 +501,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
total_tokens=total_tokens,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
batch_usage_by_model=_batch_usage_by_model(
tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
),
batch_usage_by_request=_batch_usage_by_request(
tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
),
batch_unknown_usage_requests=sum(stats is None for stats in row_stats),
),
models=(
[model_name]

View file

@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
from litellm.litellm_core_utils.model_param_helper import (
ModelParamHelper,
embeds_thought_signature_in_tool_call_id,
peek_reasoning_summary_aliases,
)
from litellm.types.caching import *
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
@ -421,7 +425,12 @@ class Cache:
reasoning_summary_material: Final = (
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
tool_call_id_shape_material: Final = (
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
)
hashed_cache_key = Cache._get_hashed_cache_key(
cache_key + reasoning_summary_material + tool_call_id_shape_material
)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",

View file

@ -432,6 +432,7 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en"
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
@ -675,7 +676,7 @@ request_timeout: float = (
DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env
)
request_timeout_explicitly_set: bool = _request_timeout_from_env is not None
DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0) # 10 minutes
DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)
AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0
AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000
# Patterns that indicate a localhost/internal URL in A2A agent cards that should be
@ -1691,6 +1692,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"

View file

@ -1554,12 +1554,66 @@ def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
def _cost_map_priced_input_output(
model: str | None,
custom_llm_provider: str | None,
usage: Usage | None,
call_type: CallTypesLiteral | None,
service_tier: str | None,
data_residency: str | None,
) -> tuple[float, float]:
if not model or usage is None:
return 0.0, 0.0
try:
return cost_per_token(
model=model,
custom_llm_provider=custom_llm_provider,
usage_object=usage,
call_type=call_type or "completion",
service_tier=service_tier,
data_residency=data_residency,
)
except Exception as pricing_error: # noqa: BLE001 # cost_per_token raises plain Exception for an unmapped model
verbose_logger.debug("No cost map price for reported-cost split of %s: %s", model, pricing_error)
return 0.0, 0.0
def split_reported_cost_by_cost_map_ratio(
reported_total: float,
model: str | None,
custom_llm_provider: str | None,
usage: Usage | None,
call_type: CallTypesLiteral | None,
service_tier: str | None,
data_residency: str | None,
) -> tuple[float, float]:
"""Apportions a provider-reported total between input and output in the ratio litellm's own cost map would
price the same usage, so a per-input-token model books the whole total on input. Unmapped or zero-priced
usage keeps the whole total on output."""
priced_input, priced_output = _cost_map_priced_input_output(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
call_type=call_type,
service_tier=service_tier,
data_residency=data_residency,
)
priced_total: Final = priced_input + priced_output
if priced_total <= 0:
return 0.0, reported_total
input_cost: Final = reported_total * (priced_input / priced_total)
return input_cost, reported_total - input_cost
def _adjusted_provider_billed_cost(
reported_cost: float,
custom_llm_provider: str | None,
pricing_provider: str | None,
model: str | None,
region_name: str | None,
litellm_logging_obj: LitellmLoggingObject | None,
usage: Usage | None,
call_type: CallTypesLiteral | None,
service_tier: str | None,
data_residency: str | None,
) -> float:
@ -1569,10 +1623,19 @@ def _adjusted_provider_billed_cost(
adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
)
input_cost, output_cost = split_reported_cost_by_cost_map_ratio(
reported_total=adjusted_cost,
model=model,
custom_llm_provider=pricing_provider,
usage=usage,
call_type=call_type,
service_tier=service_tier,
data_residency=data_residency,
)
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=adjusted_cost,
prompt_tokens_cost_usd_dollar=input_cost,
completion_tokens_cost_usd_dollar=output_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=adjusted_cost,
original_cost=reported_cost,
@ -2080,9 +2143,16 @@ def completion_cost(
return _adjusted_provider_billed_cost(
reported_cost=reported_cost,
custom_llm_provider=custom_llm_provider,
pricing_provider=_pricing_provider_for_cost_candidate(
candidate_provider=candidate_pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
model=model_for_discount,
region_name=region_name if isinstance(region_name, str) else None,
litellm_logging_obj=litellm_logging_obj,
usage=cost_per_token_usage_object,
call_type=call_type,
service_tier=service_tier,
data_residency=data_residency,
)
@ -2387,10 +2457,28 @@ def response_cost_calculator(
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
if provider_response_cost is not None and not custom_pricing:
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
header_priced_model: Final = select_model_name_for_cost_calc(
model=model,
completion_response=response_object,
custom_llm_provider=custom_llm_provider,
custom_pricing=custom_pricing,
base_model=base_model,
router_model_id=router_model_id,
region_name=region_name,
)
header_input_cost, header_output_cost = split_reported_cost_by_cost_map_ratio(
reported_total=provider_response_cost,
model=header_priced_model or model,
custom_llm_provider=pricing_provider or custom_llm_provider,
usage=get_usage_object(response_object),
call_type=call_type,
service_tier=service_tier,
data_residency=data_residency,
)
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=0.0,
completion_tokens_cost_usd_dollar=provider_response_cost,
prompt_tokens_cost_usd_dollar=header_input_cost,
completion_tokens_cost_usd_dollar=header_output_cost,
cost_for_built_in_tools_cost_usd_dollar=0.0,
total_cost_usd_dollar=provider_response_cost,
original_cost=provider_response_cost,
@ -3319,7 +3407,7 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
)
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
class _ResponsesWsEventResponse(LiteLLMBaseModel):

View file

@ -113,7 +113,7 @@ class SpeechToCompletionBridgeHandler:
voice=voice,
)
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id):
result: Final = completion(
**request_data,
)

View file

@ -13,7 +13,7 @@ import copy
import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
@ -445,22 +445,41 @@ def _as_object_list(value: object | None) -> list[object] | None:
return _validated_object_list(value)
def _tool_call_carries_cache_breakpoint(tool_call: object, message: object, answered_tool_ids: Container[str]) -> bool:
def _server_tool_call_id(tool_call: object) -> str | None:
tool_call_id: Final = _attribute_or_key(tool_call, "id")
if isinstance(tool_call_id, str) and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
return tool_call_id
return None
def _tool_call_carries_cache_breakpoint(tool_call: object, client_tool_call_ids: Container[str]) -> bool:
if _attribute_or_key(tool_call, "cache_control") is None:
return False
server_tool_call_id: Final = _server_tool_call_id(tool_call)
return server_tool_call_id is None or server_tool_call_id in client_tool_call_ids
return not tool_call_is_rebuilt_as_server_tool_use(
_attribute_or_key(tool_call, "id"), _attribute_or_key(message, "provider_specific_fields"), answered_tool_ids
def _server_tool_call_ids_without_a_kept_result(message: object) -> Iterator[str]:
provider_specific_fields: Final = _attribute_or_key(message, "provider_specific_fields")
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or ()
return (
tool_call_id
for tool_call in tool_calls
if (tool_call_id := _server_tool_call_id(tool_call)) is not None
and not tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
)
def _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
def _client_tool_call_ids_when_a_server_tool_call_is_marked(
messages: Sequence[object], model: str | None = None, custom_llm_provider: str | None = None
) -> frozenset[str]:
if not _marks_a_server_tool_call(messages):
return frozenset()
if model is not None and not _builds_anthropic_messages(model, custom_llm_provider):
return tool_call_ids_answered_in(messages)
server_calls_sent_as_client_calls: Final = itertools.chain.from_iterable(
map(_server_tool_call_ids_without_a_kept_result, messages)
)
return tool_call_ids_answered_in(messages) | frozenset(server_calls_sent_as_client_calls)
return tool_call_ids_answered_by_client(messages, _keeps_mid_conversation_system(model, custom_llm_provider))
@ -517,12 +536,7 @@ def _model_info_flags_mid_conversation_system(model: str, custom_llm_provider: s
def _is_marked_server_tool_call(tool_call: object) -> bool:
tool_call_id: Final = _attribute_or_key(tool_call, "id")
return (
isinstance(tool_call_id, str)
and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
and _attribute_or_key(tool_call, "cache_control") is not None
)
return _server_tool_call_id(tool_call) is not None and _attribute_or_key(tool_call, "cache_control") is not None
def _tool_carries_cache_breakpoint(tool: object) -> bool:
@ -668,7 +682,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
remaining_points, external_breakpoints, openai_dialect
)
answered_tool_ids: Final = _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
answered_tool_ids: Final = _client_tool_call_ids_when_a_server_tool_call_is_marked(
processed_messages, model, custom_llm_provider
)
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(
@ -757,7 +771,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
message_list: Final = tuple(messages)
client_answers: Final = (
_tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(message_list)
_client_tool_call_ids_when_a_server_tool_call_is_marked(message_list)
if answered_tool_ids is None
else answered_tool_ids
)
@ -849,7 +863,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
so earlier points win when slots are scarce.
"""
client_answers: Final = (
_tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(messages)
_client_tool_call_ids_when_a_server_tool_call_is_marked(messages)
if answered_tool_ids is None
else answered_tool_ids
)
@ -939,11 +953,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
tool_call_count: Final = (
sum(
1
for tool_call in tool_calls
if _tool_call_carries_cache_breakpoint(tool_call, message, answered_tool_ids)
)
sum(1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, answered_tool_ids))
if tool_calls
else 0
)

View file

@ -311,6 +311,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
"tool_use": "tool_calls",
"refusal": "content_filter",
"compaction": "length",
"pause_turn": "length",
"model_context_window_exceeded": "length",
# Cohere
"COMPLETE": "stop",
@ -1073,9 +1074,14 @@ def billed_usage_cost(response: object, custom_llm_provider: str | None) -> floa
_MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
def reports_native_messages_cost(custom_llm_provider: object) -> bool:
"""Whether a Messages upstream is one whose own reported cost litellm bills from instead of the price map."""
return isinstance(custom_llm_provider, str) and custom_llm_provider in _MESSAGES_REPORTED_COST_PROVIDERS
def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None:
"""Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map."""
if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS:
if not isinstance(custom_llm_provider, str) or not reports_native_messages_cost(custom_llm_provider):
return None
return provider_reported_usage_cost(usage, custom_llm_provider)

View file

@ -84,6 +84,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_host",
"github_copilot_user_session",
"fireworks_forward_user_id",
"thought_signature_in_tool_call_id",
"responses_api_version",
"allowed_openai_params",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
@ -99,6 +100,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
# Backward-compatible alias for existing imports/tests.
_OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"}
_CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
_CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
_MAX_SHOWN_INT_BITS: Final = 64

View file

@ -16,9 +16,10 @@ import traceback
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime as dt_object
from enum import Enum
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
from httpx import Response, Timeout
from pydantic import BaseModel, JsonValue, TypeAdapter
@ -47,6 +48,7 @@ from litellm.constants import (
KUBERNETES_POD_ROUTING_KEY,
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
PROVIDER_REQUEST_ID_HEADERS,
REALTIME_METERED_USAGE_KEY,
REDACTED_BY_LITELLM,
)
from litellm.cost_calculator import (
@ -84,6 +86,7 @@ from litellm.litellm_core_utils.core_helpers import (
proxy_stamped_kubernetes_pod_routing,
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
reports_native_messages_cost,
set_response_cost_in_hidden_params,
single_log_line,
)
@ -163,6 +166,7 @@ from litellm.types.llms.openai import (
ResponseIncompleteEvent,
ResponsesAPIResponse,
)
from litellm.types.llms.vertex_ai import GenerateContentResponseBody, UsageMetadata
from litellm.types.mcp import MCPPostCallResponseObject
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.rerank import RerankResponse, get_rerank_usage
@ -266,6 +270,9 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
try:
from litellm_enterprise.enterprise_callbacks.callback_controls import (
EnterpriseCallbackControls,
@ -571,6 +578,7 @@ def _resolve_mantle_region_for_cost(
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
_REDACTED_RESULT: Final = TypeAdapter(object)
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -630,7 +638,60 @@ def same_attempt_marker(first: object, second: object) -> bool:
return first is second or first == second
class UsageObservation(Enum):
KNOWN = "known"
UNKNOWN = "unknown"
ESTIMATED = "estimated"
@dataclass(frozen=True, slots=True)
class UsageCaptureObservation:
usage_json: str | None
provenance: UsageObservation
response_cost: float | None = None
@classmethod
def from_usage(
cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None
) -> "UsageCaptureObservation | UsageCaptureError":
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
try:
serialized: Final = (
None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json()
)
except Exception as error: # noqa: BLE001 # observation failures must not interrupt legacy retry handling
return UsageCaptureError(str(error))
else:
return cls(serialized, provenance, response_cost)
class Logging(LiteLLMLoggingBaseClass):
usage_capture_callback: ClassVar[
Callable[
["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None"
]
| None
] = None
def capture_usage_before_logging(
self,
response: object,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
*,
freeze_only: bool = False,
) -> None:
capture: Final = type(self).usage_capture_callback
if capture is not None and (not self.stream or self._is_assembled_stream_success(response)):
capture(
self,
response,
start_time or self.start_time,
end_time or datetime.datetime.now(datetime.timezone.utc),
freeze_only,
)
global \
supabaseClient, \
promptLayerLogger, \
@ -828,6 +889,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.finalized_cost: FinalizedCost | None = None
self.prepared_cost: PreparedCost | None = None
self.cost_allowance_discount: float | None = None
self.cost_allowance_allocation: SearchAllowanceAllocation | None = None
self.cost_finalization_lock = asyncio.Lock()
self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
self.billed_token_rates: BilledTokenRates | None = None
@ -884,7 +946,7 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.start_time = start_time
new_call.litellm_params = litellm_params
new_call.model_call_details = {
**self.model_call_details,
**{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
"litellm_call_id": litellm_call_id,
"start_time": start_time,
"first_api_call_start_time": None,
@ -894,6 +956,7 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.finalized_cost = None
new_call.prepared_cost = None
new_call.cost_allowance_discount = None
new_call.cost_allowance_allocation = None
new_call.cost_finalization_lock = asyncio.Lock()
new_call.deferred_success_result = None
new_call.streaming_chunks = []
@ -2424,7 +2487,7 @@ class Logging(LiteLLMLoggingBaseClass):
_is_sync_litellm_request = is_sync_litellm_request
def _is_assembled_stream_success(self, result=None) -> bool:
def _is_assembled_stream_success(self, result: object = None) -> bool:
"""Final assembled stream export (not a per-chunk success call).
Per-chunk callers pass a ``ModelResponseStream`` (or ``None``); the
@ -2459,7 +2522,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
from litellm.litellm_core_utils.thread_pool_executor import executor
if self._is_assembled_stream_success(result):
if self._is_assembled_stream_success(TypeAdapter(object).validate_python(result)):
if self.model_call_details.get("has_dispatched_final_stream_success"):
return
self.model_call_details["has_dispatched_final_stream_success"] = True
@ -2477,6 +2540,11 @@ class Logging(LiteLLMLoggingBaseClass):
)
return
self.capture_usage_before_logging(
TypeAdapter(object).validate_python(result),
TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time),
TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time),
)
await self.async_success_handler(
result,
start_time=start_time,
@ -2503,6 +2571,17 @@ class Logging(LiteLLMLoggingBaseClass):
return self.prepared_cost
recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
deferred: Final = self.deferred_success_result
observed_response: Final = _LITELLM_PARAMS.validate_python(
_extract_response_obj_and_hidden_params(deferred[0] if deferred else response, None)[0]
)
self.model_call_details["_accounting_usage_observed"] = (
UsageObservation.KNOWN
if (
self.model_call_details.get("combined_usage_object")
or StandardLoggingPayloadSetup.get_usage_as_dict(observed_response)
)
else UsageObservation.UNKNOWN
)
prepared: Final = self.prepare_success_logging(
result=deferred[0] if deferred is not None else response,
start_time=deferred[1] if deferred is not None else self.start_time,
@ -2540,17 +2619,29 @@ class Logging(LiteLLMLoggingBaseClass):
},
usage,
)
self.record_partial_usage_for_failure(usage, raw_cost)
self.record_partial_usage_for_failure(
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]
)
return self.prepared_cost
def record_finalized_cost(self, finalized: FinalizedCost) -> None:
self.finalized_cost = finalized
self.model_call_details["response_cost"] = finalized.amount
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
def record_usage_capture(self, captured: "CapturedUsage") -> None:
self.model_call_details["_accounting_capture"] = captured
def record_usage_pricing(self, pricing: object) -> None:
self.model_call_details["_accounting_pricing_snapshot"] = pricing
def record_partial_usage_for_failure(
self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN
) -> None:
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
if type(self).usage_capture_callback is not None:
self.model_call_details["_accounting_usage_observed"] = observation
def record_failure_after_earlier_rounds(
self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
@ -2630,6 +2721,11 @@ class Logging(LiteLLMLoggingBaseClass):
own usage while the request's failure log still bills every attempt."""
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
self.capture_failed_attempt_usage(usage)
self.model_call_details.pop("_accounting_capture", None)
self.model_call_details.pop("_accounting_pricing_snapshot", None)
self.model_call_details.pop("_accounting_usage_observed", None)
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
@ -2639,6 +2735,53 @@ class Logging(LiteLLMLoggingBaseClass):
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
)
def capture_failed_attempt_usage(self, usage: Usage) -> None:
from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
capture: Final = type(self).usage_capture_callback
if capture is None:
return
attempt: Final = self.attempt_scoped_copy()
attempt.prepared_cost = None
attempt.finalized_cost = None
identity: Final = (
f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}"
)
params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {})
metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params)
observed: Final[object] = self.model_call_details.get("_accounting_usage_observed")
cost: Final[object] = self.model_call_details.get("response_cost")
attempt.model_call_details = {
**attempt.model_call_details,
"litellm_call_id": identity,
"parent_request_id": self.litellm_call_id,
"_accounting_capture": None,
"_accounting_prior_captures": (),
"_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
usage,
observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN,
float(cost) if isinstance(cost, (int, float)) else None,
),
"litellm_params": {
**params,
"metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
"litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
},
}
captured: Final = capture(
attempt,
{"id": identity, "usage": usage.model_dump(), "status": "failed"},
self.start_time,
datetime.datetime.now(datetime.timezone.utc),
False,
)
if isinstance(captured, CapturedUsage):
self.model_call_details["_accounting_prior_captures"] = (
*self.model_call_details.get("_accounting_prior_captures", ()),
captured,
)
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
self.model_call_details["continued_partial_usage"] = usage
@ -2796,6 +2939,55 @@ class Logging(LiteLLMLoggingBaseClass):
_update_completion_start_time = update_completion_start_time
def normalize_endpoint_logging_result(self, result: object) -> object:
call_type: Final[object] = self.call_type
if call_type == CallTypes.anthropic_messages.value:
return self._anthropic_messages_logged_response(result=result)
if call_type in (CallTypes.generate_content.value, CallTypes.agenerate_content.value):
return self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
if call_type in (CallTypes.asend_message.value, CallTypes.send_message.value):
return self._handle_a2a_response_logging(result=result)
return result
def usage_capture_response(self, result: object) -> object:
if self.stream and not self._is_assembled_stream_success(result):
return None
copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result
normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied))
if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list):
frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result)
measured: Final = any(
isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping)
for frame in frames
if frame.get("type") == "response.done"
)
if not measured:
metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
return {
"id": self.litellm_call_id,
"usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
if metered is not None
else None,
}
assembled: Final = self._get_assembled_streaming_response(
normalized, self.start_time, self.start_time, True, []
)
return normalized if assembled is None else assembled
def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
from litellm.utils import count_characters
return TypeAdapter(dict[str, JsonValue]).validate_python(
{
"custom_pricing": self.custom_pricing_for(result),
"custom_cost_per_token": self.custom_cost_per_token,
"custom_cost_per_second": self.custom_cost_per_second,
"prompt_characters": count_characters(self._prompt_for_cost_calculation())
if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
else None,
}
)
def normalize_logging_result(self, result: object) -> object:
"""
Some endpoints return a different type of result than what is expected by the logging system.
@ -2806,8 +2998,20 @@ class Logging(LiteLLMLoggingBaseClass):
combined_usage_object: Final = (
RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=result)
)
metered: Final[object] = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
captured_usage: Final = (
combined_usage_object.model_copy(
update={
"metered_usage": TypeAdapter(dict[str, JsonValue]).validate_python(
TypeAdapter(object).dump_python(metered, mode="json")
)
}
)
if metered is not None
else combined_usage_object
)
logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object(
usage=combined_usage_object,
usage=captured_usage,
results=result,
)
@ -3068,7 +3272,14 @@ class Logging(LiteLLMLoggingBaseClass):
start_time = self.start_time
if end_time is None:
end_time = datetime.datetime.now()
if self.completion_start_time is None:
# A request re-dispatched to a fallback reuses this logging object but runs in
# its own window, so a first-token time from the previous attempt cannot belong
# to it. Keeping it makes the attempt report a negative TTFT.
first_token_seconds: Final = _timestamp_seconds(self.completion_start_time)
attempt_start_seconds: Final = _timestamp_seconds(start_time)
if first_token_seconds is None or (
attempt_start_seconds is not None and first_token_seconds < attempt_start_seconds
):
self.completion_start_time = end_time
self.model_call_details["completion_start_time"] = self.completion_start_time
@ -3080,15 +3291,7 @@ class Logging(LiteLLMLoggingBaseClass):
else cache_hit
)
if self.call_type == CallTypes.anthropic_messages.value:
result = self._anthropic_messages_logged_response(result=result)
elif (
self.call_type == CallTypes.generate_content.value
or self.call_type == CallTypes.agenerate_content.value
):
result = self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
elif self.call_type == CallTypes.asend_message.value or self.call_type == CallTypes.send_message.value:
result = self._handle_a2a_response_logging(result=result)
result = self.normalize_endpoint_logging_result(result)
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@ -3441,6 +3644,7 @@ class Logging(LiteLLMLoggingBaseClass):
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@ -3964,6 +4168,8 @@ class Logging(LiteLLMLoggingBaseClass):
cost_for_built_in_tools_cost_usd_dollar=0.0,
)
self.capture_usage_before_logging(result, start_time, end_time)
self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
@ -4990,16 +5196,34 @@ class Logging(LiteLLMLoggingBaseClass):
def billable_result(self, result: object) -> object:
"""The response the cost calculator prices: a /v1/messages body served through the chat
bridge is replaced by the ModelResponse its spend row is built from, anything else is
bridge is replaced by the ModelResponse its spend row is built from, a native one is only
converted when its provider reports the cost the bill is read from, anything else is
priced as is."""
served: Final[object] = result
call_type: Final[object] = getattr(self, "call_type", None)
if not isinstance(result, dict):
return served
if self.bridged_model_response is None and call_type != CallTypes.anthropic_messages.value:
if self.bridged_model_response is not None:
return self._anthropic_messages_logged_response(result=served)
if call_type != CallTypes.anthropic_messages.value:
return served
if not reports_native_messages_cost(self.model_call_details.get("custom_llm_provider")):
return self._served_with_requested_speed(result=served)
return self._anthropic_messages_logged_response(result=served)
def _served_with_requested_speed(self, result: object) -> object:
"""A native /v1/messages body priced as served bills the speed the client asked for when its
usage block reports none, the same speed its spend row is priced at."""
body: Final = _SERVED_ANTHROPIC_BODY.validate_python(result)
usage: Final = body.get("usage")
requested: Final = _LITELLM_PARAMS.validate_python(getattr(self, "optional_params", None) or {}).get("speed")
if not isinstance(requested, str) or not isinstance(usage, dict):
return result
reported: Final = _SERVED_ANTHROPIC_BODY.validate_python(usage)
if isinstance(reported.get("speed"), str):
return result
return {**body, "usage": {**reported, "speed": requested}}
def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
"""
The ModelResponse a /v1/messages spend_logs row is built from.
@ -6808,45 +7032,19 @@ class StandardLoggingPayloadSetup:
@staticmethod
def get_usage_from_response_obj(response_obj: dict | None, combined_usage_object: Usage | None = None) -> Usage:
## BASE CASE ##
if combined_usage_object is not None:
return combined_usage_object
if response_obj is None:
return Usage(
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
)
usage: Final = response_obj.get("usage", None)
if usage is None:
rerank_usage: Final = get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj))
if rerank_usage is not None:
return Usage(**rerank_usage)
if usage is None or (not isinstance(usage, dict) and not isinstance(usage, Usage)):
return Usage(
prompt_tokens=0,
completion_tokens=0,
total_tokens=0,
)
elif isinstance(usage, Usage):
return usage
elif isinstance(usage, ResponseAPIUsage):
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
elif isinstance(usage, dict):
if ResponseAPILoggingUtils.is_response_api_usage(usage):
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
return Usage(**usage)
raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
reported: Final = response_obj.get("usage") if response_obj else None
if isinstance(reported, Usage):
return reported
normalized: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj)
return Usage(**normalized) if normalized else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
@staticmethod
def get_usage_as_dict(
response_obj: dict | None,
response_obj: dict[str, Any] | None,
combined_usage_object: Usage | None = None,
) -> dict:
) -> dict[str, Any]:
"""
Like get_usage_from_response_obj but returns a plain dict, skipping
the Pydantic Usage construction on the hot path.
@ -6858,7 +7056,21 @@ class StandardLoggingPayloadSetup:
return _empty
_raw: Final = response_obj.get("usage", None)
if _raw is None:
return dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
google_usage: Final[object] = response_obj.get("usageMetadata")
if isinstance(google_usage, dict) and _GOOGLE_USAGE_FIELDS.intersection(google_usage):
return litellm.VertexGeminiConfig.calculate_usage(
cast( # cast-ok: retain converter-supported modalities and token_count aliases
GenerateContentResponseBody, response_obj
)
).model_dump()
reported: Final = response_obj.get("usage_info")
if isinstance(reported, BaseModel):
return reported.model_dump(exclude_none=True)
return (
{key: value for key, value in reported.items() if value is not None}
if isinstance(reported, dict)
else dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
)
if isinstance(_raw, ResponseAPIUsage):
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
if isinstance(_raw, dict):

View file

@ -17,6 +17,7 @@ from openai.types.responses.response_create_params import (
ResponseCreateParamsNonStreaming,
ResponseCreateParamsStreaming,
)
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.types.llms.anthropic import AnthropicMessagesRequest
@ -41,6 +42,16 @@ def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> obj
return None
_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None)
def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool:
try:
return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False
except ValidationError:
return True
class ModelParamHelper:
# Cached at class level — deterministic set built from static OpenAI type annotations
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

View file

@ -238,7 +238,7 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
return any(message.get(key, None) is not None for key in message if key not in ignore_keys)
_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
_IMAGE_CONTENT_PART_TYPES: Final = ("image_url", "input_image", "image", "computer_screenshot")
_IMAGE_SCAN_MAX_DEPTH: Final = 4
@ -557,15 +557,24 @@ def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObjec
return {"type": "image_url", "image_url": {"url": url}} if url else None
def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
"""Whether any message carries an image content part, across the dialects that reach
pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,
and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks."""
return any(
isinstance(content, list) and _content_parts_contain_image(content)
for message in messages
for content in (message.get("content"),)
)
def _message_contains_image_content(message: object) -> bool:
match message:
case {"content": list()}:
return _content_parts_contain_image(content_parts(_CONTENT_PART.validate_python(message)))
case {"type": "function_call_output" | "custom_tool_call_output" | "tool_result", "output": list()}:
output: Final = _CONTENT_PART.validate_python(message)["output"]
return _content_parts_contain_image(_CONTENT_PARTS.validate_python(output))
case {"type": "computer_call_output", "output": dict()}:
return _content_parts_contain_image((_CONTENT_PART.validate_python(message)["output"],))
case _:
return False
def request_contains_image_content(messages: object) -> bool:
"""Inspect native message content without translating or validating the request."""
if not isinstance(messages, (list, tuple)):
return False
return any(_message_contains_image_content(message) for message in _CONTENT_PARTS.validate_python(messages))
def audio_or_image_in_message_content(message: AllMessageValues) -> bool:

View file

@ -26,6 +26,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
is_anthropic_server_tool_use,
is_anthropic_tool_search_use,
message_field,
parts_of,
@ -1252,6 +1253,22 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str
_encode_tool_call_id_with_signature = encode_tool_call_id_with_signature
def _tool_call_id_prefix(tool_call_id: object) -> str | None:
return tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] if isinstance(tool_call_id, str) else None
def _called_tool_name(tool_call_id: str, tool_calls: Sequence[Mapping[str, object]]) -> str | None:
exact_matches: Final = tuple(call for call in tool_calls if call.get("id") == tool_call_id)
prefix: Final = _tool_call_id_prefix(tool_call_id)
prefix_matches: Final = tuple(
call for call in tool_calls if prefix and _tool_call_id_prefix(call.get("id")) == prefix
)
matches: Final = exact_matches or (prefix_matches if len(prefix_matches) == 1 else ())
function: Final = matches[0].get("function") if matches else None
name: Final = function.get("name") if isinstance(function, Mapping) else None
return name if isinstance(name, str) else None
def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
"""Extract thought signature from tool call's provider_specific_fields.
@ -1691,16 +1708,14 @@ def convert_to_gemini_tool_call_result(
)
) is not None:
media_parts.append(media_part)
name: str | None = message.get("name", "")
# Recover name from last message with tool calls
if last_message_with_tool_calls:
tools: Final = last_message_with_tool_calls.get("tool_calls", [])
msg_tool_call_id: Final = message.get("tool_call_id", None)
for tool in tools:
prev_tool_call_id = tool.get("id", None)
if msg_tool_call_id and prev_tool_call_id and msg_tool_call_id == prev_tool_call_id:
name = tool.get("function", {}).get("name", "")
msg_tool_call_id: Final = message.get("tool_call_id", None)
called_tool_name: Final = (
_called_tool_name(msg_tool_call_id, last_message_with_tool_calls.get("tool_calls") or ())
if last_message_with_tool_calls and isinstance(msg_tool_call_id, str) and msg_tool_call_id
else None
)
name: Final[str | None] = called_tool_name if called_tool_name is not None else message.get("name", "")
# Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
gemini_call_id: str | None = None
@ -2059,6 +2074,54 @@ def _placed_and_paired(
_AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
def _is_server_tool_result(block: Mapping[str, object]) -> bool:
block_type: Final = block.get("type")
return isinstance(block_type, str) and block_type.endswith("_tool_result")
def _split_unrun_server_tool_uses(
groups: Sequence[Sequence[_AnthropicToolInvokeItem]],
) -> tuple[tuple[Sequence[_AnthropicToolInvokeItem], ...], tuple[_AnthropicToolInvokeItem, ...]]:
completed: Final = tuple(group for group in groups if any(_is_server_tool_result(block) for block in group))
unrun: Final = tuple(group[0] for group in groups if not any(_is_server_tool_result(block) for block in group))
return completed, unrun
def _block_id(block: Mapping[str, object]) -> object:
return block.get("id")
def _unrun_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozenset[object]:
result_ids: Final = frozenset(block.get("tool_use_id") for block in blocks if _is_server_tool_result(block))
return frozenset(
_block_id(block)
for block in blocks
if block.get("type") == "server_tool_use" and _block_id(block) not in result_ids
)
def _last_block_id_among(blocks: Sequence[Mapping[str, object]], ids: frozenset[object]) -> object:
return next(block_id for block in reversed(blocks) if (block_id := _block_id(block)) in ids)
def _paused_turn_replay(
tool_invokes: Sequence[_AnthropicToolInvokeItem],
thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
*,
is_final_message: bool,
) -> tuple[
tuple[_AnthropicToolInvokeItem, ...],
tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...],
]:
unrun_ids: Final = _unrun_server_tool_use_ids(tool_invokes)
if not unrun_ids:
return tuple(tool_invokes), tuple(thinking_blocks)
dropped_ids: Final = (
unrun_ids - frozenset((_last_block_id_among(tool_invokes, unrun_ids),)) if is_final_message else unrun_ids
)
return tuple(block for block in tool_invokes if _block_id(block) not in dropped_ids), ()
def convert_to_anthropic_tool_invoke(
tool_calls: list[ChatCompletionAssistantToolCall],
web_search_results: Sequence[object] | None = None,
@ -2141,6 +2204,15 @@ def convert_to_anthropic_tool_invoke(
anthropic_tool_invoke.append(server_tool_result)
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
continue
elif is_anthropic_server_tool_use(tool_id) and tool_id not in answered_tool_ids:
anthropic_tool_invoke.append(
{
"type": "server_tool_use",
"id": tool_id,
"name": tool_name,
"input": tool_input,
}
)
else:
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
@ -3109,6 +3181,30 @@ def anthropic_messages_pt(
_has_server_tool_calls = True
break
# Get web_search_results and tool_results from provider_specific_fields
# for server_tool_use reconstruction.
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields: dict[str, Any] = (
cast(dict[str, Any], _provider_specific_fields_raw)
if isinstance(_provider_specific_fields_raw, dict)
else {}
)
tool_invoke_results = (
convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_provider_specific_fields.get("web_search_results"),
tool_results=_provider_specific_fields.get("tool_results"),
answered_tool_ids=client_answered_tool_ids,
)
if assistant_tool_calls is not None
else ()
)
turn_paused = bool(_unrun_server_tool_use_ids(tool_invoke_results))
replayed_tool_invokes, replayed_thinking_blocks = _paused_turn_replay(
tool_invoke_results, thinking_blocks or (), is_final_message=msg_i == len(messages) - 1
)
if (
thinking_blocks is not None
and _has_server_tool_calls
@ -3121,26 +3217,12 @@ def anthropic_messages_pt(
# We must preserve this interleaved order because Anthropic
# verifies thinking block signatures based on position.
# Build the tool call groups (server_tool_use + its result)
_provider_specific_fields_raw_tc = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields_tc: dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw_tc, dict):
_provider_specific_fields_tc = cast(dict[str, Any], _provider_specific_fields_raw_tc)
_web_search_results_tc = _provider_specific_fields_tc.get("web_search_results")
_tool_results_tc = _provider_specific_fields_tc.get("tool_results")
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results_tc,
tool_results=_tool_results_tc,
answered_tool_ids=client_answered_tool_ids,
)
# Group tool invoke results into (server_tool_use, result) pairs
# and separate regular tool_use blocks
server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
regular_tool_uses: list[_AnthropicToolInvokeItem] = []
_current_group: list[_AnthropicToolInvokeItem] = []
for item in tool_invoke_results:
for item in replayed_tool_invokes:
item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
if item_type == "server_tool_use":
if _current_group:
@ -3154,6 +3236,9 @@ def anthropic_messages_pt(
_current_group.append(item)
if _current_group:
server_tool_groups.append(_current_group)
completed_server_tool_groups, pending_server_tool_uses = _split_unrun_server_tool_uses(
server_tool_groups
)
# Build the text block if content is a non-empty string
text_element = None
@ -3177,15 +3262,15 @@ def anthropic_messages_pt(
# Any remaining groups (after all thinking blocks) go after.
tb_idx = 0
grp_idx = 0
num_tb = len(thinking_blocks) if thinking_blocks else 0
num_grp = len(server_tool_groups)
num_tb = len(replayed_thinking_blocks)
num_grp = len(completed_server_tool_groups)
while tb_idx < num_tb or grp_idx < num_grp:
if tb_idx < num_tb and grp_idx < num_grp:
# Emit thinking block then its tool group
assistant_content.append(thinking_blocks[tb_idx])
assistant_content.append(replayed_thinking_blocks[tb_idx])
tb_idx += 1
for block in server_tool_groups[grp_idx]:
for block in completed_server_tool_groups[grp_idx]:
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3195,11 +3280,11 @@ def anthropic_messages_pt(
grp_idx += 1
elif tb_idx < num_tb:
# More thinking blocks than tool groups - emit before text
assistant_content.append(thinking_blocks[tb_idx])
assistant_content.append(replayed_thinking_blocks[tb_idx])
tb_idx += 1
else:
# More tool groups than thinking blocks - emit remaining
for block in server_tool_groups[grp_idx]:
for block in completed_server_tool_groups[grp_idx]:
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3212,8 +3297,8 @@ def anthropic_messages_pt(
if text_element is not None:
assistant_content.append(text_element)
# Add regular (non-server) tool calls at the end
for item in regular_tool_uses:
# Add regular tool calls, then a paused turn's unrun server tool call, at the end
for item in (*regular_tool_uses, *pending_server_tool_uses):
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3233,18 +3318,18 @@ def anthropic_messages_pt(
)
_content_list = assistant_content_block.get("content") if _content_is_list else None
if (
thinking_blocks is not None and not _content_is_list
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(thinking_blocks)
if not _content_is_list: # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(replayed_thinking_blocks)
if _content_is_list and _content_list is not None:
for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()):
for m in _with_missing_thinking_blocks(_content_list, replayed_thinking_blocks):
if not isinstance(m, dict):
continue
# handle thinking blocks
if (thinking_block := _as_thinking_block(m)) is not None:
assistant_content.extend(
_replayable_thinking_blocks([thinking_block], drop_params=drop_params)
()
if turn_paused
else _replayable_thinking_blocks([thinking_block], drop_params=drop_params)
)
continue
text_block = cast(str, m.get("text", ""))
@ -3284,25 +3369,9 @@ def anthropic_messages_pt(
assistant_content.append(_anthropic_text_content_element)
if assistant_tool_calls is not None: # support assistant tool invoke conversion
# Get web_search_results and tool_results from provider_specific_fields
# for server_tool_use reconstruction.
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields: dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw, dict):
_provider_specific_fields = cast(dict[str, Any], _provider_specific_fields_raw)
_web_search_results = _provider_specific_fields.get("web_search_results")
_tool_results = _provider_specific_fields.get("tool_results")
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results,
tool_results=_tool_results,
answered_tool_ids=client_answered_tool_ids,
)
# Prevent "tool_use ids must be unique" errors by filtering duplicates
# This can happen when merging history that already contains the tool calls
for item in tool_invoke_results:
for item in replayed_tool_invokes:
# tool_use items are typically dicts, but handle objects just in case
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)

View file

@ -286,6 +286,10 @@ def _separate_thinking_blocks_render(message: object) -> bool:
return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks")))
def is_anthropic_server_tool_use(tool_id: str) -> bool:
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES

View file

@ -631,6 +631,8 @@ class RealTimeStreaming:
if self.session_tools or self.tool_calls:
self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
if LiteLLMLogging.usage_capture_callback is not None:
self.logging_obj.capture_usage_before_logging(self.messages)
# Route through the bounded logging worker (per-coroutine timeout +
# concurrency cap) instead of a bare create_task, so a slow callback
# can't leave suspended tasks pinning each call's response in memory.

View file

@ -1509,11 +1509,20 @@ def _delta_reasoning_items(delta: object) -> Sequence[object]:
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
def _chunk_first_delta(chunk: object) -> object:
choices: Final = _field_of(chunk, "choices")
if not isinstance(choices, list) or not choices:
return ()
return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk
return None
return _field_of(cast(Sequence[object], choices)[0], "delta") # cast-ok: untyped chunk
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
return _delta_reasoning_items(_chunk_first_delta(chunk))
def _chunk_refusal(chunk: object) -> str:
refusal: Final = _field_of(_chunk_first_delta(chunk), "refusal")
return refusal if isinstance(refusal, str) else ""
_NON_TEXT_DELTA_FIELDS: Final = (
@ -1526,6 +1535,7 @@ _NON_TEXT_DELTA_FIELDS: Final = (
"audio",
"images",
"provider_specific_fields",
"refusal",
)
@ -1674,6 +1684,9 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
combined_refusal: Final = "".join(_chunk_refusal(chunk) for chunk in typed_chunks)
if combined_refusal:
yield "refusal", combined_refusal
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
yield "reasoning_items", list(reasoning_items)

View file

@ -349,11 +349,14 @@ class CustomStreamWrapper:
make_call: Callable | None = None,
_response_headers: dict | httpx.Headers | None = None,
count_prompt_tokens: Callable[[], int] | None = None,
is_mock: bool = False,
):
self.model = model
self.make_call = make_call
self.count_prompt_tokens = count_prompt_tokens
self.custom_llm_provider = custom_llm_provider
self.is_mock = is_mock
self.uses_openai_chat_stream = False
self.logging_obj: LiteLLMLoggingObject = logging_obj
self.completion_stream = completion_stream
self.sent_first_chunk = False
@ -376,6 +379,7 @@ class CustomStreamWrapper:
self._upstream_provider: str | None = None
self.received_finish_reason: str | None = None
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason
self.special_tokens = [
"<|assistant|>",
"<|system|>",
@ -1302,8 +1306,8 @@ class CustomStreamWrapper:
if _is_delta_empty:
model_response.choices[0].delta = Delta(content=self._close_open_think_block())
# get any function call arguments
model_response.choices[0].finish_reason = map_finish_reason(
finish_reason=self.received_finish_reason
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
map_finish_reason(finish_reason=self.received_finish_reason)
) # ensure consistent output to openai
self.sent_last_chunk = True
@ -1597,6 +1601,7 @@ class CustomStreamWrapper:
if response_obj["is_finished"]:
self.received_finish_reason = response_obj["finish_reason"]
else: # openai / azure chat model
self.uses_openai_chat_stream = True
if isinstance(chunk, ChatCompletionChunk):
self._openai_handler_receives_chat_chunks = True
if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
@ -1617,6 +1622,7 @@ class CustomStreamWrapper:
return _ProviderChunkEarlyReturn(None)
completion_obj["content"] = response_obj["text"]
self.intermittent_finish_reason = response_obj.get("finish_reason", None)
self.received_any_finish_reason |= response_obj["finish_reason"] is not None
if response_obj["is_finished"]:
if response_obj["finish_reason"] == "error":
raise Exception(
@ -1737,6 +1743,8 @@ class CustomStreamWrapper:
# if function returned but type set to None - mistral's api returns type: None
tool["type"] = "function"
model_response.choices[0].delta = Delta(**_json_delta)
if model_response.choices[0].delta.tool_calls:
self.tool_call = True
except Exception as e:
verbose_logger.exception(
"litellm.CustomStreamWrapper.chunk_creator(): Exception occured - %s", e
@ -1966,6 +1974,11 @@ class CustomStreamWrapper:
if self.logging_obj.is_sync_litellm_request(litellm_params):
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
def _normalize_tool_call_finish_reason(self, finish_reason: str) -> str:
if finish_reason == "stop" and self.tool_call:
return "tool_calls"
return finish_reason
def finish_reason_handler(self):
model_response: Final = self.model_response_creator(
hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
@ -1981,21 +1994,30 @@ class CustomStreamWrapper:
if served_tier is not None:
setattr(model_response, "service_tier", served_tier) # noqa: B010 # pydantic extra, not a declared field
_finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
if _finish_reason is not None:
model_response.choices[0].finish_reason = _finish_reason
else:
model_response.choices[0].finish_reason = "stop"
## if tool use
if (
model_response.choices[0].finish_reason == "stop" and self.tool_call
): # don't overwrite for other - potential error finish reasons
model_response.choices[0].finish_reason = "tool_calls"
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
_finish_reason if _finish_reason is not None else "stop"
)
close_tag: Final = self._close_open_think_block()
if close_tag is not None:
model_response.choices[0].delta.content = close_tag
return model_response
def _should_fail_on_clean_eof(self) -> bool:
return (
not self.is_mock
and self.uses_openai_chat_stream
and self.received_finish_reason is None
and self.intermittent_finish_reason is None
and not self.received_any_finish_reason
)
@staticmethod
def _stream_eof_error() -> httpx.RemoteProtocolError:
return httpx.RemoteProtocolError(
"Stream ended before a finish_reason was received",
request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs"),
)
def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
return processed_chunk
@ -2181,6 +2203,13 @@ class CustomStreamWrapper:
return response
except StopIteration:
if self._should_fail_on_clean_eof():
error: Final = self._stream_eof_error()
eof_traceback: Final = traceback.format_exc()
self._record_partial_usage_for_failure()
threading.Thread(target=self.logging_obj.failure_handler, args=(error, eof_traceback)).start()
self.close()
self._handle_stream_fallback_error(error)
if self.sent_last_chunk is True:
complete_streaming_response: Final = self._build_complete_streaming_response()
response = self._new_usage_chunk()
@ -2384,6 +2413,9 @@ class CustomStreamWrapper:
continue
return usage_free_chunk
except (StopAsyncIteration, StopIteration):
if self._should_fail_on_clean_eof():
await self.aclose()
self._log_stream_failure_and_raise(self._stream_eof_error())
return await self._finalize_completed_stream(cache_hit=cache_hit)
except httpx.TimeoutException as e: # if httpx read timeout error occues
traceback_exception = traceback.format_exc()

View file

@ -160,8 +160,11 @@ async def resolve_a2a_hop_auth_header(
litellm_params: Mapping[str, object],
custom_llm_provider: object,
resolve_entra_header: AgentAuthHeaderResolver = resolve_azure_ai_agent_auth_header,
agent_extra_headers: Mapping[str, str] | None = None,
) -> Mapping[str, str] | None:
"""Entra credentials authenticate the A2A hop only; a completion-bridge agent hands them to the model provider it bridges to."""
if not a2a_hop_uses_entra(litellm_params, custom_llm_provider):
if a2a_hop_uses_entra(litellm_params, custom_llm_provider):
return await resolve_entra_header(litellm_params)
if custom_llm_provider or any(name.lower() == "authorization" for name in (agent_extra_headers or {})):
return None
return await resolve_entra_header(litellm_params)
api_key: Final = litellm_params.get("api_key")
return {"Authorization": f"Bearer {api_key}"} if isinstance(api_key, str) and api_key else None

View file

@ -70,7 +70,12 @@ from litellm.types.utils import (
)
from ...base import BaseLLM
from ..common_utils import AnthropicError, process_anthropic_headers, refusal_provider_specific_fields
from ..common_utils import (
AnthropicError,
process_anthropic_headers,
refusal_params,
stop_reason_provider_specific_fields,
)
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
if TYPE_CHECKING:
@ -926,6 +931,17 @@ class ModelResponseIterator:
def _web_search_call_snapshot(self) -> dict[str, object]:
return dict(self._web_search_calls)
def _refresh_pending_web_search_call(self, tool_use_id: str) -> bool:
if tool_use_id not in self._web_search_calls:
return False
self._web_search_calls[tool_use_id] = build_web_search_call(
tool_use_id,
self._server_tool_inputs.get(tool_use_id, {}),
{"content": []},
status="in_progress",
)
return True
def _complete_web_search_call(self, result: dict[str, object]) -> None:
tool_use_id: Final = result.get("tool_use_id")
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
@ -1128,6 +1144,8 @@ class ModelResponseIterator:
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
except (json.JSONDecodeError, TypeError):
pass
if self._refresh_pending_web_search_call(self._current_server_tool_id):
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
self._current_server_tool_id = None
if self.is_response_format_tool:
text = self._flush_response_format_buffer()
@ -1140,8 +1158,8 @@ class ModelResponseIterator:
# These are automatically handled by Anthropic API, we just pass them through
pass
elif type_chunk == "message_delta":
finish_reason, usage, container, refusal_fields = self._handle_message_delta(chunk)
provider_specific_fields.update(refusal_fields)
finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk)
provider_specific_fields.update(stop_reason_fields)
text = self._release_answer_close() + self._drain_split_answer_blocks()
if container:
provider_specific_fields["container"] = container
@ -1190,6 +1208,7 @@ class ModelResponseIterator:
)
text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
refusal_fields: Final = refusal_params(provider_specific_fields)
returned_chunk: Final = ModelResponseStream(
choices=[
@ -1201,6 +1220,7 @@ class ModelResponseIterator:
provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
thinking_blocks=(thinking_blocks if thinking_blocks else None),
reasoning_content=reasoning_content,
**refusal_fields,
),
finish_reason=finish_reason,
)
@ -1371,7 +1391,7 @@ class ModelResponseIterator:
self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) if "usage" in message_delta else None
)
container: Final = message_delta["delta"].get("container")
return finish_reason, usage, container, refusal_provider_specific_fields(message_delta["delta"])
return finish_reason, usage, container, stop_reason_provider_specific_fields(message_delta["delta"])
def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
"""

View file

@ -120,13 +120,14 @@ from litellm.utils import (
)
from ..common_utils import (
REFUSAL_STOP_FIELDS,
STOP_REASON_FIELDS,
AnthropicError,
AnthropicModelInfo,
eager_input_streaming_flag,
process_anthropic_headers,
refusal_provider_specific_fields,
refusal_params,
requires_native_compaction_beta,
stop_reason_provider_specific_fields,
strip_advisor_blocks_from_messages,
tools_marked_for_a_tool_config_point,
)
@ -167,6 +168,8 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
@ -2919,28 +2922,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
web_search_results: Sequence[object],
completion_response: Mapping[str, object],
) -> list[object]:
content: Final = completion_response.get("content")
blocks: Final = content if isinstance(content, Sequence) else ()
inputs: Final = {
call_id: tool_input
for block in blocks
if isinstance(block, Mapping)
and block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
and isinstance((tool_input := block.get("input")), Mapping)
blocks: Final = _RESPONSE_BLOCKS.validate_python(completion_response.get("content") or ())
results_by_id: Final = {
tool_use_id: result
for result in _RESPONSE_BLOCKS.validate_python(web_search_results)
if result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
}
return [
build_web_search_call(
tool_id=tool_use_id,
tool_input=inputs.get(tool_use_id, {}),
result=result,
tool_id=call_id,
tool_input=block.get("input"),
result=results_by_id.get(call_id, _PENDING_WEB_SEARCH_RESULT),
status=None if call_id in results_by_id else "in_progress",
)
for result in web_search_results
if isinstance(result, dict)
and result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
and tool_use_id in inputs
for block in blocks
if block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
]
def _build_provider_specific_fields(
@ -2964,10 +2963,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if web_search_results is not None:
provider_specific_fields["web_search_results"] = web_search_results
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
web_search_results,
completion_response,
)
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
if web_search_results is not None or web_search_calls:
provider_specific_fields["web_search_calls"] = web_search_calls
if tool_results is not None:
provider_specific_fields["tool_results"] = tool_results
@ -2989,10 +2988,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if compaction_blocks is not None:
provider_specific_fields["compaction_blocks"] = compaction_blocks
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
stop_reason_fields: Final = stop_reason_provider_specific_fields(
STOP_REASON_FIELDS.validate_python(completion_response)
)
return {**provider_specific_fields, **refusal_fields}
return {**provider_specific_fields, **stop_reason_fields}
def transform_parsed_response(
self,
@ -3067,22 +3066,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if json_extra_content:
merged_text = merged_text + json_extra_content if merged_text else json_extra_content
refusal_fields: Final = refusal_params(STOP_REASON_FIELDS.validate_python(completion_response))
_message = litellm.Message(
tool_calls=tool_calls_for_message,
content=merged_text or None,
provider_specific_fields=provider_specific_fields,
thinking_blocks=thinking_blocks,
reasoning_content=reasoning_content,
**refusal_fields,
)
_message.provider_specific_fields = provider_specific_fields
if json_mode_message is not None:
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
stop_reason_fields: Final = stop_reason_provider_specific_fields(
STOP_REASON_FIELDS.validate_python(completion_response)
)
completion_response["stop_reason"] = "stop"
if refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if stop_reason_fields:
json_mode_message.provider_specific_fields = dict(stop_reason_fields)
if "refusal" in refusal_fields:
json_mode_message.refusal = refusal_fields["refusal"]
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks
json_mode_message.reasoning_content = reasoning_content

View file

@ -2518,7 +2518,7 @@ def create_anthropic_model_list_response(
}
class RefusalStopFields(TypedDict, total=False):
class StopReasonFields(TypedDict, total=False):
stop_reason: ReadOnly[object]
stop_details: ReadOnly[object]
@ -2535,16 +2535,38 @@ def _dict_or_empty(value: object) -> object:
return value if _is_dict(value) else _no_stop_fields()
REFUSAL_STOP_FIELDS: Final[TypeAdapter[RefusalStopFields]] = TypeAdapter(
Annotated[RefusalStopFields, BeforeValidator(_dict_or_empty)]
STOP_REASON_FIELDS: Final[TypeAdapter[StopReasonFields]] = TypeAdapter(
Annotated[StopReasonFields, BeforeValidator(_dict_or_empty)]
)
_STOP_REASONS_KEPT_RAW: Final = frozenset({"refusal", "pause_turn"})
def refusal_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
def stop_reason_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
stop_reason: Final = message.get("stop_reason")
stop_details: Final = message.get("stop_details")
if stop_details is not None:
return {"stop_reason": stop_reason, "stop_details": stop_details}
if stop_reason == "refusal":
if stop_reason in _STOP_REASONS_KEPT_RAW:
return {"stop_reason": stop_reason}
return {}
class _RefusalStopDetails(TypedDict, total=False):
explanation: ReadOnly[object]
_REFUSAL_STOP_DETAILS: Final[TypeAdapter[_RefusalStopDetails]] = TypeAdapter(
Annotated[_RefusalStopDetails, BeforeValidator(_dict_or_empty)]
)
class RefusalParams(TypedDict, total=False):
refusal: ReadOnly[str]
def refusal_params(message: Mapping[str, object]) -> RefusalParams:
if message.get("stop_reason") != "refusal":
return {}
explanation: Final = _REFUSAL_STOP_DETAILS.validate_python(message.get("stop_details")).get("explanation")
return {"refusal": explanation} if isinstance(explanation, str) and explanation else {}

View file

@ -118,9 +118,10 @@ def _extract_proxy_litellm_metadata(
def _requested_router(kwargs: Mapping[str, object]) -> "Router | None":
from litellm.router import Router
from litellm.router_python_core import PythonCore
requested: Final = kwargs.get("litellm_router")
return requested if isinstance(requested, Router) else None
return requested if isinstance(requested, (Router, PythonCore)) else None # pyright: ignore[reportReturnType] # the router passes its core, which carries Router's members
def request_safeguards_evaluator(

View file

@ -33,6 +33,7 @@ from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
AnthropicStopDetails,
AppliedEdit,
CompactionBlock,
ContentBlockDelta,
@ -284,9 +285,7 @@ class _CombinedChunkSplitter:
@staticmethod
def _is_combined(chunk: "ModelResponseStream") -> bool:
"""True if ``chunk`` carries response content AND a finish_reason."""
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
choices: Final = _optional_attr_sequence(chunk, "choices")
if not choices:
@ -302,7 +301,7 @@ class _CombinedChunkSplitter:
or _optional_attr(delta, "tool_calls")
or _optional_attr(delta, "reasoning_content")
or _optional_attr(delta, "thinking_blocks")
or openai_chat_refusal_text(delta)
or refusal_reply_text(delta)
)
_PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@ -659,6 +658,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
self.compaction_block = compaction_block
self.iterations_usage = iterations_usage
self._refusal_text: str = ""
self._paused_turn: bool = False
self._claude_refusal_stop_details: AnthropicStopDetails | None = None
self.sent_compaction_block: bool = False
# Per-phase flags so the compaction block's start/delta/stop events
# are emitted (and the public state machine is advanced) in
@ -953,7 +954,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
current_content_block_index=self.current_content_block_index,
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
)
processed_chunk = self._with_refusal_stop_details(processed_chunk)
processed_chunk = self._with_stop_reason_fields(processed_chunk)
# Check if this is a usage chunk and we have a held stop_reason chunk
if will_merge_into_held:
@ -1188,7 +1189,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
current_content_block_index=self.current_content_block_index,
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
)
processed_chunk = self._with_refusal_stop_details(processed_chunk)
processed_chunk = self._with_stop_reason_fields(processed_chunk)
# Check if this is a usage chunk and we have a held stop_reason chunk
if will_merge_into_held:
@ -1406,14 +1407,22 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
def _increment_content_block_index(self):
self.current_content_block_index += 1
def _with_refusal_stop_details(
def _with_stop_reason_fields(
self,
processed_chunk: ContentBlockDelta | MessageBlockDelta,
) -> ContentBlockDelta | MessageBlockDelta:
if processed_chunk["type"] != "message_delta" or not self._refusal_text:
if processed_chunk["type"] != "message_delta":
return processed_chunk
delta: Final = processed_chunk["delta"]
if delta.get("stop_reason") == "max_tokens":
if self._paused_turn:
paused_delta: Final[MessageDelta] = {**delta, "stop_reason": "pause_turn"}
paused_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": paused_delta}
return paused_chunk
if delta.get("stop_reason") == "refusal" and self._claude_refusal_stop_details is not None:
claude_refusal_delta: Final[MessageDelta] = {**delta, "stop_details": self._claude_refusal_stop_details}
claude_refusal_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": claude_refusal_delta}
return claude_refusal_chunk
if not self._refusal_text or delta.get("stop_reason") == "max_tokens":
return processed_chunk
from litellm.llms.anthropic.messages.native.utils import (
refusal_stop_details,
@ -1470,9 +1479,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
@staticmethod
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
choice: Final = chunk.choices[0]
if choice.finish_reason is not None:
@ -1482,7 +1489,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
if getattr(delta, "content", None):
return False
if openai_chat_refusal_text(delta):
if refusal_reply_text(delta):
return False
if getattr(delta, "reasoning_content", None):
return False
@ -1497,6 +1504,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
return True
def _record_anthropic_stop_fields(self, delta: object) -> None:
from litellm.llms.anthropic.messages.native.utils import (
anthropic_refusal_stop_details,
anthropic_stop_reason,
)
if anthropic_stop_reason(delta) == "pause_turn":
self._paused_turn = True
claude_refusal_stop_details: Final = anthropic_refusal_stop_details(delta)
if claude_refusal_stop_details is not None:
self._claude_refusal_stop_details = claude_refusal_stop_details
def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
"""
Determine if we should start a new content block based on the processed chunk.
@ -1507,16 +1526,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
- Different content types in the response
- Specific markers in the content
"""
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
from .transformation import LiteLLMAnthropicMessagesAdapter
self._record_anthropic_stop_fields(chunk.choices[0].delta)
if chunk.choices[0].finish_reason is not None:
return False
refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
refusal_text: Final = refusal_reply_text(chunk.choices[0].delta)
if refusal_text is not None:
self._refusal_text = self._refusal_text + refusal_text

View file

@ -150,7 +150,9 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import (
is_system_role_message,
)
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
anthropic_refusal_stop_details,
anthropic_stop_reason,
refusal_reply_text,
refusal_stop_details,
)
from litellm.types.llms.anthropic import (
@ -398,6 +400,10 @@ class LiteLLMAnthropicMessagesAdapter:
return None
def _streamed_tool_call_signature(self, tool_call_id: str, tool_call: object) -> str | None:
id_parts: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
return id_parts[1] if len(id_parts) > 1 else self._extract_signature_from_tool_call(tool_call)
def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None:
"""
Extract signature from a tool_use content block's provider_specific_fields.
@ -1467,7 +1473,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_content.append(
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
)
if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
if (refusal_text := refusal_reply_text(choice.message)) is not None:
new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
# Handle tool calls (in parallel to text content)
if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
@ -1634,7 +1640,15 @@ class LiteLLMAnthropicMessagesAdapter:
tool_name_mapping=tool_name_mapping,
)
refusal_text: Final = next(
(text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
(text for choice in response.choices if (text := refusal_reply_text(choice.message)) is not None),
None,
)
claude_refusal_stop_details: Final = next(
(
details
for choice in response.choices
if (details := anthropic_refusal_stop_details(choice.message)) is not None
),
None,
)
@ -1643,8 +1657,10 @@ class LiteLLMAnthropicMessagesAdapter:
## extract finish reason
openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop"
translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
openai_finish_reason=openai_finish_reason
translated_finish_reason: Final = (
"pause_turn"
if response.choices and anthropic_stop_reason(response.choices[0].message) == "pause_turn"
else self._translate_openai_finish_reason_to_anthropic(openai_finish_reason=openai_finish_reason)
)
anthropic_finish_reason: Final = (
"compaction"
@ -1686,7 +1702,11 @@ class LiteLLMAnthropicMessagesAdapter:
usage=anthropic_usage,
content=anthropic_content,
stop_reason=anthropic_finish_reason,
stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
stop_details=(
(claude_refusal_stop_details or refusal_stop_details(refusal_text))
if anthropic_finish_reason == "refusal"
else None
),
)
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
@ -1712,10 +1732,7 @@ class LiteLLMAnthropicMessagesAdapter:
):
raw_id = choice.delta.tool_calls[0].id or str(uuid.uuid4())
tool_name = choice.delta.tool_calls[0].function.name or ""
thought_sig: str | None = None
if THOUGHT_SIGNATURE_SEPARATOR in raw_id:
parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
thought_sig = parts[1] if len(parts) > 1 else None
thought_sig = self._streamed_tool_call_signature(raw_id, choice.delta.tool_calls[0])
tool_block: dict[str, object] = {
"type": "tool_use",
"id": normalize_anthropic_tool_use_id(raw_id),
@ -1727,7 +1744,7 @@ class LiteLLMAnthropicMessagesAdapter:
"signature": thought_sig,
}
return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or refusal_reply_text(
choice.delta
) is not None:
return "text", TextBlock(type="text", text="")
@ -1810,7 +1827,7 @@ class LiteLLMAnthropicMessagesAdapter:
return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
else:
refusal_text: Final = "".join(
refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
refusal for choice in choices if (refusal := refusal_reply_text(choice.delta)) is not None
)
return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)

View file

@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Final, cast, get_type_hints
from pydantic import JsonValue, TypeAdapter, ValidationError
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import as_str_mapping
@ -162,6 +162,34 @@ def openai_chat_refusal_text(message_or_delta: object) -> str | None:
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
def anthropic_stop_reason(message_or_delta: object) -> str | None:
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "stop_reason")
def refusal_reply_text(message_or_delta: object) -> str | None:
if anthropic_stop_reason(message_or_delta) == "refusal":
return None
return openai_chat_refusal_text(message_or_delta)
@with_config(ConfigDict(extra="allow"))
class _AnthropicRefusalStopDetails(AnthropicStopDetails, total=False):
pass
_ANTHROPIC_REFUSAL_STOP_DETAILS: Final = TypeAdapter(_AnthropicRefusalStopDetails)
def anthropic_refusal_stop_details(message_or_delta: object) -> AnthropicStopDetails | None:
provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None)
if _mapping_str_field(provider_specific_fields, "stop_reason") != "refusal":
return None
try:
return _ANTHROPIC_REFUSAL_STOP_DETAILS.validate_python(_mapping_field(provider_specific_fields, "stop_details"))
except ValidationError:
return refusal_stop_details(openai_chat_refusal_text(message_or_delta))
def _responses_message_refusal_text(item: object) -> str | None:
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal

View file

@ -155,6 +155,8 @@ def collapse_pure_text_chunks(
continue
json_str = ev[idx + len("data:") :].strip()
if not json_str.startswith("{"):
continue
try:
data = json.loads(json_str)
except (json.JSONDecodeError, ValueError):
@ -206,6 +208,11 @@ def collapse_pure_text_chunks(
return out
def _has_non_object_data(event_str: str) -> bool:
data_start: Final = event_str.find("data:")
return data_start != -1 and not event_str[data_start + len("data:") :].lstrip().startswith("{")
def build_complete_streaming_response_legacy(
all_chunks: Sequence[str | bytes],
litellm_logging_obj: LiteLLMLoggingObj,
@ -243,6 +250,8 @@ def build_complete_streaming_response_legacy(
# text payload happens to contain "[DONE]" is not dropped.
if any(line.strip() == "data: [DONE]" for line in event_str.split("\n")):
continue
if _has_non_object_data(event_str):
continue
transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk(
chunk=event_str
)
@ -268,21 +277,25 @@ def build_complete_streaming_response_legacy(
return complete_streaming_response
def extract_sse_data(event_str: str) -> dict | None:
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
def _sse_data_payload(event_str: str) -> str | None:
for line in split_sse_lines(event_str):
stripped = line.strip()
if stripped.startswith("data:"):
payload = stripped[len("data:") :].strip()
if not payload or payload == "[DONE]":
return None
try:
return json.loads(payload)
except (ValueError, TypeError):
return None
return stripped[len("data:") :].strip()
return None
def extract_sse_data(event_str: str) -> dict | None:
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
payload: Final = _sse_data_payload(event_str)
if payload is None or not payload.startswith("{"):
return None
try:
return json.loads(payload)
except (ValueError, TypeError):
return None
def build_usage_only_response_from_chunks(
all_chunks: Sequence[str | bytes],
model: str,
@ -415,27 +428,81 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset(
)
_ERROR_PAYLOAD_KEYS: Final = frozenset({"type", "error", "request_id"})
def _is_anthropic_event_type(event_type: object) -> bool:
return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool:
return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks)
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
raw: Final = b"".join(
def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes:
return b"".join(
chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
for chunk in all_chunks
if isinstance(chunk, (str, bytes))
)
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
try:
return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False)
return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False)
except UnicodeDecodeError:
return None
def _sse_event_data(event: str) -> Mapping[str, object] | None:
return as_str_mapping(extract_sse_data(event))
_JSON_ESCAPED_LINE_BREAKS: Final = MappingProxyType(
str.maketrans({"\x85": "\\u0085", "\u2028": "\\u2028", "\u2029": "\\u2029"})
)
def _sse_frames(sse_stream: str) -> Sequence[str]:
return "\n".join(split_sse_lines(sse_stream)).split("\n\n")
def _frames_of_the_widest_reader(sse_stream: str) -> Sequence[str]:
"""The stream's frames as httpx's ``str.splitlines`` reads them, past a form feed or U+2028 in any line."""
return "\n".join(sse_stream.splitlines()).split("\n\n")
def _sse_field(line: str) -> tuple[str, str]:
name, _, value = line.partition(":")
return name, value.removeprefix(" ")
def _sse_fields(frame: str) -> tuple[tuple[str, str], ...]:
"""The frame's fields as a client reads them: a name matches exactly and a comment is no field."""
return tuple(_sse_field(line) for line in frame.split("\n") if line and not line.startswith(":"))
def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
match fields:
case () | (("event", _),) | (("data", "[DONE]"),):
return True
case _:
return False
def _json_mapping(payload: str) -> Mapping[str, object] | None:
try:
value: Final[object] = json.loads(payload)
except ValueError:
return None
return as_str_mapping(value)
def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]:
return tuple(
event_data
for event in split_sse_chunk_into_events(sse_stream)
if (event_data := extract_sse_data(event)) is not None
if (event_data := _sse_event_data(event)) is not None
)
@ -444,7 +511,8 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
(
message
for event_data in _parsed_sse_events(sse_stream)
if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict)
if event_data.get("type") == "message_start"
and (message := as_str_mapping(event_data.get("message"))) is not None
),
None,
)
@ -482,11 +550,151 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return False
return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream))
return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream))
def _is_json_no_reader_can_take(payload: str) -> bool:
try:
value: Final[object] = json.loads(payload)
except ValueError:
return False
if value is None:
return False
event: Final = as_str_mapping(value)
return event is None or isinstance(event.get("type"), (list, dict))
def has_unreadable_json_event(all_chunks: Sequence[object]) -> bool:
"""Whether a frame's data is JSON that no reader of these streams can take as an event."""
sse_stream: Final = codecs.getincrementaldecoder("utf-8")(errors="replace").decode(
_joined_sse_bytes(all_chunks), final=False
)
return any(_is_json_no_reader_can_take(_sse_data_payload(event) or "") for event in _sse_frames(sse_stream))
def anthropic_sse_event_type(event: str) -> object:
return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None)
def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
"""The string ``type`` of every event the stream sent."""
event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
return frozenset(event_type for event_type in event_types if isinstance(event_type, str))
def _is_named_error_event(event: str) -> bool:
names: Final = tuple(value for name, value in _sse_fields(event) if name == "event")
return names[-1:] == ("error",)
def _is_error_event(event: str) -> bool:
event_type: Final = anthropic_sse_event_type(event)
return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type))
def _frames_a_client_dispatches(sse_stream: str) -> Sequence[str]:
return "\n".join((*split_sse_lines(sse_stream), "")).split("\n\n")[:-1]
def _error_events(sse_stream: str) -> tuple[str, ...]:
return tuple(event for event in _frames_a_client_dispatches(sse_stream) if _is_error_event(event))
def _plain_error_event(event: str) -> str:
payload: Final = _sse_data_payload(event) or ""
one_line: Final = payload.translate(_JSON_ESCAPED_LINE_BREAKS) if _json_mapping(payload) is not None else payload
return "\n".join(("event: error", *(f"data: {line}" for line in one_line.splitlines()[:1])))
def anthropic_sse_error_events(all_chunks: Sequence[object]) -> tuple[str, ...]:
"""The ``error`` events of the stream, each as an ``event:`` line and the ``data:`` line assembly read.
A stream whose upstream fails after it started ends with an ``error`` event in place of
``message_stop``. That frame is the outcome the client is owed, not content, so assembly through
the upstream error leaves it out and a caller re-emitting a rewritten response puts it back,
without any other line of the frame, since that is content no guardrail read.
A frame counts by its data ``type``, or by its SSE ``event:`` name unless its data is another
Anthropic event, since a client SDK raises on the name alone and an upstream that is not
Anthropic may send anything as the data.
"""
return tuple(_plain_error_event(event) for event in _error_events(_joined_sse_stream(all_chunks) or ""))
def anthropic_sse_stopped_block_index(event: str) -> object:
return next(
(
event_data.get("index")
for event_data in _parsed_sse_events(event)
if event_data.get("type") == "content_block_stop"
),
None,
)
def _block_index(event_data: Mapping[str, object]) -> int | None:
index: Final = event_data.get("index")
return index if isinstance(index, int) else None
def _started_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
starts: Final = (event for event in events if event.get("type") == "content_block_start")
return MappingProxyType(
{
index: block
for event in starts
if (index := _block_index(event)) is not None
and (block := as_str_mapping(event.get("content_block"))) is not None
}
)
def anthropic_sse_started_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
"""The ``content_block`` each ``content_block_start`` of the stream opened, by block index."""
return _started_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
def _unfinished_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
stopped: Final = frozenset(_block_index(event) for event in events if event.get("type") == "content_block_stop")
return MappingProxyType({index: block for index, block in _started_blocks(events).items() if index not in stopped})
def anthropic_sse_unfinished_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
"""The blocks the stream opened and never stopped, which only a failed upstream leaves."""
return _unfinished_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
def _sse_events_with_data(sse_stream: str) -> tuple[tuple[str, Mapping[str, object]], ...]:
return tuple(
(event, event_data)
for event in split_sse_chunk_into_events(sse_stream)
if (event_data := _sse_event_data(event)) is not None
)
def _content_events_before_upstream_error(sse_stream: str) -> tuple[str, ...]:
"""The stream's events minus its upstream ``error`` and every cut-off block assembly cannot read.
A cut-off text block is still text the client was sent, so it stays. A cut-off tool call's
input is a JSON prefix, and a cut-off thinking block has no signature, so neither is a block the
client can act on, and both are left out.
"""
if not _error_events(sse_stream):
return (sse_stream,)
events: Final = _sse_events_with_data(sse_stream)
cut_off: Final = frozenset(
index
for index, block in _unfinished_blocks(tuple(event_data for _, event_data in events)).items()
if block.get("type") != "text"
)
return tuple(
event
for event, event_data in events
if event_data.get("type") != "error" and _block_index(event_data) not in cut_off
)
def assemble_anthropic_sse_stream(
all_chunks: Sequence[object], *, restore_identity: bool = False
all_chunks: Sequence[object], *, restore_identity: bool = False, through_upstream_error: bool = False
) -> ModelResponse | None:
"""Assemble raw Anthropic SSE frames into a ModelResponse.
@ -494,6 +702,11 @@ def assemble_anthropic_sse_stream(
assembler does not carry through. It is off by default so callers that re-emit the assembled
response keep the wire shape they had before this helper was shared. The writes land on a
freshly built object that is unreachable from caller state until returned.
A stream whose upstream failed after it started ends with an ``error`` event in place of
``message_stop`` and assembles to None, unless ``through_upstream_error`` asks for the content
the client was sent before the failure. That is for a guardrail that scans the stream and then
passes the upstream error on; a caller recording what was served keeps None.
"""
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
@ -502,9 +715,12 @@ def assemble_anthropic_sse_stream(
if message_start is None:
return None
model: Final = message_start.get("model") if restore_identity else None
content_events: Final = (
_content_events_before_upstream_error(sse_stream) if through_upstream_error else (sse_stream,)
)
try:
assembled: Final = build_complete_streaming_response(
all_chunks=(sse_stream,),
all_chunks=content_events,
litellm_logging_obj=None, # pyright: ignore[reportArgumentType] # only forwarded to stream_chunk_builder, which accepts None
model=model if isinstance(model, str) else "",
)
@ -522,6 +738,21 @@ def assemble_anthropic_sse_stream(
return assembled
def _is_error_payload(payload: str) -> bool:
event: Final = _json_mapping(payload)
if event is None or not _ERROR_PAYLOAD_KEYS.issuperset(event):
return False
return event.get("type") == "error" or ("type" not in event and isinstance(event.get("error"), Mapping))
def _is_error_frame(fields: Sequence[tuple[str, str]]) -> bool:
match fields:
case (("data", payload),) | (("event", "error"), ("data", payload)):
return _is_error_payload(payload)
case _:
return False
def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
"""Whether the buffered stream carries nothing but error frames.
@ -537,7 +768,6 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return False
events: Final = _parsed_sse_events(sse_stream)
return len(events) > 0 and all(
event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
)
frames: Final = tuple(_sse_fields(frame) for frame in _frames_of_the_widest_reader(sse_stream))
with_data: Final = tuple(fields for fields in frames if not _carries_no_data(fields))
return len(with_data) > 0 and all(_is_error_frame(fields) for fields in with_data)

View file

@ -544,6 +544,17 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
return AgentCoreParsedResponse(content=content, usage=usage_data, final_message=final_message)
@staticmethod
def _message_stop_chunk(model: str) -> ModelResponseStream:
chunk: Final = ModelResponseStream(
id=f"chatcmpl-{uuid.uuid4()}",
created=0,
model=model,
object="chat.completion.chunk",
)
chunk.choices = [StreamingChoices(finish_reason="stop", index=0, delta=Delta())]
return chunk
def _stream_agentcore_response_sync(
self,
response: httpx.Response,
@ -553,6 +564,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
Internal sync generator that parses SSE and yields ModelResponse chunks.
"""
buffer = ""
sent_finish = False # rebind-ok: flips when a finish chunk goes out
message_stopped = False # rebind-ok: flips on messageStop, the only end some runtimes send
for text_chunk in response.iter_text():
buffer += text_chunk
@ -624,7 +637,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
total_tokens=usage_data.get("totalTokens", 0),
),
)
sent_finish = True
yield chunk
message_stopped |= "messageStop" in event_payload
# Process final message
if "message" in data_obj and isinstance(data_obj["message"], dict):
@ -641,12 +656,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
delta=Delta(),
)
]
sent_finish = True
yield chunk
except json.JSONDecodeError:
verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
continue
if message_stopped and not sent_finish:
yield self._message_stop_chunk(model)
def get_sync_custom_stream_wrapper(
self,
model: str,
@ -775,6 +794,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
Internal async generator that parses SSE and yields ModelResponse chunks.
"""
buffer = ""
sent_finish = False # rebind-ok: flips when a finish chunk goes out
message_stopped = False # rebind-ok: flips on messageStop, the only end some runtimes send
async for text_chunk in response.aiter_text():
buffer += text_chunk
@ -846,7 +867,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
total_tokens=usage_data.get("totalTokens", 0),
),
)
sent_finish = True
yield chunk
message_stopped |= "messageStop" in event_payload
# Process final message
if "message" in data_obj and isinstance(data_obj["message"], dict):
@ -863,12 +886,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
delta=Delta(),
)
]
sent_finish = True
yield chunk
except json.JSONDecodeError:
verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
continue
if message_stopped and not sent_finish:
yield self._message_stop_chunk(model)
async def get_async_custom_stream_wrapper(
self,
model: str,

View file

@ -131,6 +131,8 @@ from litellm.responses.streaming_iterator import (
ResponsesAPIStreamingIterator,
ResponsesWebSocketStreaming,
SyncResponsesAPIStreamingIterator,
TokenLimitCallback,
with_token_limit_checks,
)
from litellm.types.containers.main import (
ContainerFileListResponse,
@ -598,10 +600,10 @@ def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
)
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
"""Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
enforcement, so the Responses WebSocket loop can charge every
``response.create`` frame, not just the connection's first one.
def _ws_callbacks_exposing(method_name: str) -> tuple[object, ...]:
"""Duck-type discover proxy hooks exposing a per-frame check, so the
Responses WebSocket loop can run it on every ``response.create`` frame,
not just the connection's first one.
Uses duck-typing on ``litellm.callbacks`` (rather than importing the
proxy hook directly) to avoid a layering violation (SDK importing from
@ -612,13 +614,28 @@ def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
callbacks: Final = cast( # cast-ok: callback registry is inspected before protocol use
Sequence[object], _litellm.callbacks
)
return tuple(callback for callback in callbacks if callable(getattr(callback, method_name, None)))
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
return tuple(
cast(ProjectQuotaCallback, callback) # cast-ok: required callback method is callable
for callback in callbacks
if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
for callback in _ws_callbacks_exposing("enforce_project_io_token_quota_for_frame")
)
def _collect_ws_token_limit_callbacks() -> tuple[TokenLimitCallback, ...]:
return tuple(
cast(TokenLimitCallback, callback) # cast-ok: required callback method is callable
for callback in _ws_callbacks_exposing("enforce_token_limits_for_frame")
)
def _ws_model_group(litellm_metadata: Mapping[str, object] | None, model: str) -> str:
model_group: Final = (litellm_metadata or {}).get("model_group")
return model_group if isinstance(model_group, str) and model_group else model
def _unwrap_provider_error(error: Exception) -> Exception:
if not isinstance(error, httpx.TransportError):
return error
@ -7285,6 +7302,12 @@ class BaseLLMHTTPHandler:
- Forwards events over the websocket connection
"""
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
_ws_turn_admission: Final = with_token_limit_checks(
turn_admission,
_collect_ws_token_limit_callbacks(),
user_api_key_dict,
_ws_model_group(litellm_metadata, model),
)
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
@ -7332,7 +7355,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
first_message=first_message,
quota_callbacks=_ws_quota_callbacks,
turn_admission=turn_admission,
turn_admission=_ws_turn_admission,
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
frame_guard=frame_guard,
**kwargs,
@ -7429,7 +7452,7 @@ class BaseLLMHTTPHandler:
authorized_model=model,
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
turn_admission=turn_admission,
turn_admission=_ws_turn_admission,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

View file

@ -1,16 +1,21 @@
import json
import math
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from urllib.parse import parse_qs, urlparse
from urllib.parse import parse_qs, urlparse, urlsplit, urlunsplit
import httpx
import litellm
from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL
from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL, DEEPGRAM_SPEAK_DEFAULT_MODEL
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.passthrough_endpoints.pass_through_endpoints import WebSocketSessionUsage
from litellm.types.utils import LlmProviders
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
_WEBSOCKET_SCHEMES: Final = MappingProxyType({"https": "wss", "http": "ws", "wss": "wss", "ws": "ws"})
DEEPGRAM_LISTEN_CALLBACK_PARAMS: Final = frozenset({"callback", "callback_method"})
@ -199,3 +204,71 @@ def deepgram_listen_transcript(websocket_messages: Sequence[Mapping[str, object]
if frame.get("type") == "Results"
if (transcript := _final_transcript(frame)) is not None
)
_AURA_GENERATION: Final = re.compile(r"aura(?:-\d+)?(?=-[a-z])")
def deepgram_speak_websocket_target(api_base: str | None, query_string: str) -> str:
speak_url: Final = urlsplit(join_api_base(api_base or DEEPGRAM_DEFAULT_API_BASE, "speak"))
scheme: Final = _WEBSOCKET_SCHEMES.get(speak_url.scheme, speak_url.scheme)
params: Final = _first_occurrences(query_string)
query: Final = params if params.get("model") else params.remove("model").add("model", DEEPGRAM_SPEAK_DEFAULT_MODEL)
return urlunsplit((scheme, speak_url.netloc, speak_url.path, str(query), ""))
def deepgram_speak_model(upstream_url: str) -> str:
models: Final = parse_qs(urlparse(upstream_url).query).get("model")
return models[0] if models else DEEPGRAM_SPEAK_DEFAULT_MODEL
def deepgram_speak_requested_model(query_string: str) -> str:
"""Read off the upstream target, so the voice a key is authorized for is the voice Deepgram receives"""
return deepgram_speak_model(deepgram_speak_websocket_target(None, query_string))
def deepgram_speak_registry_key(upstream_url: str) -> str:
"""Deepgram bills every voice of an Aura generation at one rate, so ``aura-2-thalia-en`` is priced at the
``deepgram/aura-2`` row"""
model: Final = deepgram_speak_model(upstream_url)
generation: Final = _AURA_GENERATION.match(model)
return f"{LlmProviders.DEEPGRAM.value}/{generation.group() if generation else model}"
def deepgram_speak_character_rate(upstream_url: str) -> float | None:
"""Only an exact registry hit counts, so a voice is never billed at a neighbouring row's rate"""
registry_key: Final = deepgram_speak_registry_key(upstream_url)
model_info: Final = lookup_model_info_for_registration(registry_key, LlmProviders.DEEPGRAM.value)
if isinstance(model_info, ModelInfoLookupMiss) or model_info["key"] != registry_key:
return None
return model_info.get("input_cost_per_character")
@dataclass(frozen=True, slots=True)
class DeepgramSpeakMeter:
"""Holds the rate checked when the socket opened, so a cost-map reload mid-session can't change or drop the price"""
upstream_url: str
character_rate: float
def measure(self, client_text: str) -> int:
"""Counts a Speak message's text as it is sent. Deepgram also speaks, and bills, unflushed text on its own (a
long buffer, Close, a close frame), so counting only flushed text would let a client get audio free, while
counting on send can only over-bill text Deepgram discards, such as a Clear or an oversized Speak. Never
raises, so a malformed frame can't end the session"""
try:
message: Final[object] = json.loads(client_text)
except (ValueError, RecursionError):
return 0
match message:
case {"type": "Speak", "text": str() as text}:
return len(text)
case _:
return 0
def price(self, total: int) -> WebSocketSessionUsage:
return WebSocketSessionUsage(
model=deepgram_speak_model(self.upstream_url),
custom_llm_provider=LlmProviders.DEEPGRAM.value,
response_cost=total * self.character_rate,
)

View file

@ -41,8 +41,9 @@ class LangGraphSSEStreamIterator:
return self
def __aiter__(self):
"""Initialize async iteration."""
self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
"""Initialize async iteration once; CustomStreamWrapper re-enters `async for` on every chunk."""
if self.async_line_iterator is None:
self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
return self
def _parse_sse_line(self, line: str) -> ModelResponseStream | None:

View file

@ -20,7 +20,11 @@ import litellm
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.litellm_core_utils.prompt_templates.common_utils import (
merge_consecutive_system_messages,
messages_with_file_parts_as_image_urls,
system_messages_first,
)
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@ -693,14 +697,18 @@ class OCIChatConfig(BaseConfig):
chatRequest=chat_request,
)
else:
generic_messages: Final = list(messages_with_file_parts_as_image_urls(messages))
normalized_messages: Final = (
merge_consecutive_system_messages(system_messages_first(generic_messages))
if model.lower().removeprefix("oci/").startswith("google.gemini")
else generic_messages
)
data = OCICompletionPayload(
compartmentId=oci_compartment_id,
servingMode=serving_mode,
chatRequest=OCIChatRequestPayload(
apiFormat=vendor.value,
messages=adapt_messages_to_generic_oci_standard(
list(messages_with_file_parts_as_image_urls(messages))
),
messages=adapt_messages_to_generic_oci_standard(normalized_messages),
**self._get_optional_params(vendor, optional_params, model),
),
)

View file

@ -38,8 +38,13 @@ from itertools import accumulate, chain, groupby, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import (
ReadOnly,
TypedDict,
TypeIs, # noqa: TID251 # TypeIs reaches typing only on 3.13
)
from litellm._logging import verbose_proxy_logger
from litellm.completion_extras.litellm_responses_transformation.transformation import (
@ -99,6 +104,7 @@ from litellm.types.llms.openai import (
OutputTextDeltaEvent,
OutputTextDoneEvent,
ResponseAPIUsage,
ResponseCompactionItem,
ResponseCompletedEvent,
ResponsesAPIOptionalRequestParams,
ResponsesAPIResponse,
@ -277,6 +283,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
)
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
_TEXT_DELTA_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.refusal.delta"})
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
@ -537,6 +544,14 @@ def _scanned_text_field(item: Mapping[str, object]) -> str:
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
def _content_part_text_field(part: object) -> str:
return "refusal" if stream_item_field(part, "type") == "refusal" else "text"
def _has_writable_field(part: object, field: str) -> bool:
return isinstance(part, dict) or hasattr(part, field)
def _tool_output_part_text(part: object) -> str | None:
if not isinstance(part, dict):
return None
@ -702,6 +717,15 @@ def _next_stream_sequence_number(responses_so_far: Sequence[object] | None) -> i
return next((n + 1 for n in sequence_numbers if isinstance(n, int)), 0)
def _is_responses_input(
input_data: object,
) -> TypeIs["str | ResponseInputParam"]: # guard-ok: str, or a list of Mapping items
if isinstance(input_data, str):
return True
items: Final = cast(list[object], input_data) # cast-ok: isinstance narrows to list[Unknown] only
return isinstance(input_data, list) and all(isinstance(item, (Mapping, ResponseCompactionItem)) for item in items)
class OpenAIResponsesHandler(BaseTranslation):
"""
Handler for processing OpenAI Responses API with guardrails.
@ -723,8 +747,8 @@ class OpenAIResponsesHandler(BaseTranslation):
Transforms `input` (string or ResponseInputParam) and optional
`instructions` into chat completion messages.
"""
input_data: Final = data.get("input")
if input_data is None:
input_data: Final[object] = data.get("input")
if not _is_responses_input(input_data):
return None
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
input=input_data,
@ -993,7 +1017,7 @@ class OpenAIResponsesHandler(BaseTranslation):
for content_idx, content_item in enumerate(content):
if isinstance(content_item, dict):
# Extract text
text_str = content_item.get("text", None)
text_str = content_item.get(_content_part_text_field(content_item), None)
if text_str is not None:
texts_to_check.append(text_str)
task_mappings.append((msg_idx, int(content_idx)))
@ -1042,8 +1066,7 @@ class OpenAIResponsesHandler(BaseTranslation):
elif isinstance(content, list):
# Replace specific text item in list content
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
self._write_content_part_text(content[content_idx_optional], guardrail_response)
async def process_output_response(
self,
@ -1380,7 +1403,8 @@ class OpenAIResponsesHandler(BaseTranslation):
if isinstance(output_index, int) and isinstance(content_index, int)
)
if len(placeable_positions) != len(scanned_positions) or any(
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES for event in scanned_events
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES | _TEXT_DELTA_EVENT_TYPES
for event in scanned_events
):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
@ -1401,6 +1425,12 @@ class OpenAIResponsesHandler(BaseTranslation):
else:
setattr(event, field, value)
@staticmethod
def _write_content_part_text(part: object, text: str) -> None:
field: Final = _content_part_text_field(part)
if _has_writable_field(part, field):
OpenAIResponsesHandler._write_event_field(part, field, text)
@staticmethod
def _write_compaction_summary(item: object, summary: str) -> None:
encrypted_content: Final = stream_item_field(item, "encrypted_content")
@ -1416,11 +1446,12 @@ class OpenAIResponsesHandler(BaseTranslation):
) -> None:
"""Sync pre-completion stream events with the rewritten completed
response, keyed by ``(output_index, content_index)``: the first
``output_text.delta`` for a rewritten item carries the full rewritten
text and the rest are blanked, while ``output_text.done``,
``content_part.done``, and ``output_item.done`` events carry the full
rewritten text, so every event a client may read agrees with the
rewritten ``response.completed`` payload."""
``output_text.delta`` or ``refusal.delta`` for a rewritten item carries
the full rewritten text and the rest are blanked, while
``output_text.done``, ``refusal.done``, ``content_part.done``, and
``output_item.done`` events carry the full rewritten text, so every
event a client may read agrees with the rewritten ``response.completed``
payload."""
delta_replacements: Final = MappingProxyType(
{position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()}
)
@ -1436,14 +1467,14 @@ class OpenAIResponsesHandler(BaseTranslation):
if not isinstance(output_index, int) or not isinstance(content_index, int):
continue
position = (output_index, content_index)
if event_type == "response.output_text.delta" and position in delta_replacements:
if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements:
self._write_event_field(event, "delta", next(delta_replacements[position]))
elif event_type == "response.output_text.done" and position in rewrites_by_position:
self._write_event_field(event, "text", rewrites_by_position[position])
elif event_type == "response.refusal.done" and position in rewrites_by_position:
self._write_event_field(event, "refusal", rewrites_by_position[position])
elif event_type == "response.content_part.done" and position in rewrites_by_position:
part = event.get("part")
if isinstance(part, dict) or hasattr(part, "text"):
self._write_event_field(part, "text", rewrites_by_position[position])
self._write_content_part_text(event.get("part"), rewrites_by_position[position])
@staticmethod
def _sync_output_item_done_event(
@ -1459,7 +1490,7 @@ class OpenAIResponsesHandler(BaseTranslation):
for (item_idx, content_idx), rewritten in rewrites_by_position.items():
if item_idx != output_index or not 0 <= content_idx < len(content):
continue
OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten)
OpenAIResponsesHandler._write_content_part_text(content[content_idx], rewritten)
def _deliver_ended_stream_tool_call_rewrites(
self,
@ -1814,7 +1845,7 @@ class OpenAIResponsesHandler(BaseTranslation):
return
# Handle both GenericResponseOutputItem and dict
content: list[OutputText] | list[dict] | None = None
content: list[OutputText | ResponseOutputRefusal] | list[dict[str, object]] | None = None
if isinstance(output_item, BaseModel):
try:
output_item_dump: Final = output_item.model_dump()
@ -1837,17 +1868,9 @@ class OpenAIResponsesHandler(BaseTranslation):
verbose_proxy_logger.debug("OpenAI Responses API: Processing output item: %s", output_item)
# Iterate through content items (list of OutputText objects)
for content_idx, content_item in enumerate(content):
# Handle both OutputText objects and dicts
if isinstance(content_item, OutputText):
text_content = content_item.text
elif isinstance(content_item, dict):
text_content = content_item.get("text")
else:
continue
if text_content:
text_content = stream_item_field(content_item, _content_part_text_field(content_item))
if isinstance(text_content, str) and text_content:
texts_to_check.append(text_content)
task_mappings.append((output_idx, int(content_idx)))
@ -1887,33 +1910,24 @@ class OpenAIResponsesHandler(BaseTranslation):
# Handle both GenericResponseOutputItem, BaseModel, and dict
if isinstance(output_item, GenericResponseOutputItem):
if output_item.content and content_idx < len(output_item.content):
content_item = output_item.content[content_idx]
if isinstance(content_item, OutputText):
content_item.text = guardrail_response
elif isinstance(content_item, dict):
content_item["text"] = guardrail_response
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
elif isinstance(output_item, BaseModel):
# Handle other Pydantic models by converting to GenericResponseOutputItem
try:
generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())
if generic_item.content and content_idx < len(generic_item.content):
content_item = generic_item.content[content_idx]
if isinstance(content_item, OutputText):
content_item.text = guardrail_response
# Update the original response output
if hasattr(output_item, "content") and output_item.content:
original_content = output_item.content[content_idx]
if hasattr(original_content, "text"):
original_content.text = guardrail_response
if (
generic_item.content
and content_idx < len(generic_item.content)
and hasattr(output_item, "content")
and output_item.content
):
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
except Exception:
pass
elif isinstance(output_item, dict):
content = output_item.get("content", [])
if content and content_idx < len(content):
if isinstance(content[content_idx], dict):
content[content_idx]["text"] = guardrail_response
elif hasattr(content[content_idx], "text"):
content[content_idx].text = guardrail_response
self._write_content_part_text(content[content_idx], guardrail_response)
def build_block_sse_chunks(
self,

View file

@ -129,6 +129,17 @@ def _native_vertex_row_usage(
return calculate_usage(completion_response)
def native_vertex_batch_row_model(row: Mapping[str, object], model_name: str | None) -> str | None:
response: Final = as_json_object(row.get("response"))
model_version: Final = (
response.get("model" if is_vertex_anthropic_batch_output_response(response) else "modelVersion")
if response is not None
else None
)
deployment_model: Final = model_name if model_name and "*" not in model_name else None
return deployment_model or (model_version if isinstance(model_version, str) else model_name)
def native_vertex_batch_row_stats(
row: Mapping[str, object],
model_name: str | None,
@ -156,11 +167,7 @@ def native_vertex_batch_row_stats(
if usage is None:
return None
total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
model_version: Final = response_body.get(
"model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
)
deployment_model: Final = model_name if model_name and "*" not in model_name else None
model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
model: Final = native_vertex_batch_row_model(row, model_name)
if model is None:
verbose_logger.warning(
"vertex_ai batch output row could not be costed, so it is billed at $0 and the rest of the batch "

View file

@ -31,9 +31,8 @@ from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
from litellm.litellm_core_utils.prompt_templates.factory import (
encode_tool_call_id_with_signature,
)
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -2091,6 +2090,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
parts: list[HttpxPartType],
cumulative_tool_call_idx: int,
is_function_call: bool | None,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[
ChatCompletionToolCallFunctionChunk | None,
list[ChatCompletionToolCallChunk] | None,
@ -2129,9 +2129,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# back on the matching `functionResponse`.
if gemini_call_id:
_tool_response_chunk["id"] = gemini_call_id
# Embed thought signature in ID for OpenAI client compatibility
if thought_signature:
_tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature}
if thought_signature and thought_signature_in_tool_call_id:
_tool_response_chunk["id"] = encode_tool_call_id_with_signature(
_tool_response_chunk["id"] or "", thought_signature
)
@ -2793,6 +2793,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
model_response: Union[ModelResponse, "ModelResponseStream"],
standard_optional_params: dict,
cumulative_tool_call_index: int = 0,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[list[dict], list[dict], list, list, int]:
"""
Helper method to process candidates and extract metadata
@ -2895,6 +2896,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
parts=candidate["content"]["parts"],
cumulative_tool_call_idx=cumulative_tool_call_index,
is_function_call=is_function_call(standard_optional_params),
thought_signature_in_tool_call_id=thought_signature_in_tool_call_id,
)
if "logprobsResult" in candidate:
@ -2978,6 +2980,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
model_response: Union[ModelResponse, "ModelResponseStream"],
standard_optional_params: dict[str, object], # mutable-ok: mirrors override contract
cumulative_tool_call_index: int = 0,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[ # mutable-ok: mirrors override contract
list[dict[str, object]],
list[dict[str, object]],
@ -2986,7 +2989,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
int,
]:
return cls._process_candidates(
_candidates, model_response, standard_optional_params, cumulative_tool_call_index
_candidates,
model_response,
standard_optional_params,
cumulative_tool_call_index,
thought_signature_in_tool_call_id,
)
def transform_response(
@ -3069,7 +3076,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
safety_ratings,
citation_metadata,
_, # cumulative_tool_call_index not needed in non-streaming
) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params)
) = VertexGeminiConfig._process_candidates(
_candidates,
model_response,
logging_obj.optional_params,
thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id(
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
),
)
usage: Final = VertexGeminiConfig._calculate_usage(
completion_response=completion_response,
@ -3905,6 +3919,9 @@ class ModelResponseIterator:
self.logging_obj = logging_obj
self.response_headers = response_headers or {}
self.is_function_call = check_is_function_call(logging_obj)
self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id(
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
)
self.cumulative_tool_call_index: int = 0
self.has_seen_tool_calls: bool = False
self._resume = resume
@ -3969,6 +3986,7 @@ class ModelResponseIterator:
model_response,
self.logging_obj.optional_params,
cumulative_tool_call_index=self.cumulative_tool_call_index,
thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id,
)
# Track whether tool_calls have been seen across streaming chunks.

View file

@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin
from concurrent import futures
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from copy import deepcopy
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
@ -82,8 +83,8 @@ from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_litellm_params import (
AWS_CREDENTIAL_KWARGS_KEYS,
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
OPTIONAL_KWARGS_KEYS,
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
SUPPLEMENTAL_KWARGS_KEYS,
InvalidControlOption,
parse_control_options,
with_control_options,
@ -157,7 +158,7 @@ from litellm.types.completion import (
)
from litellm.types.litellm_params import ControlOptions, RetryStrategy
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly
from litellm.types.utils import (
AgenticLoopParams,
BaseUrlRoutingHint,
@ -399,6 +400,49 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
return assert_never(control)
@dataclass(frozen=True, slots=True)
class _SupplementalProviderParams:
params: GenericLiteLLMParams
dropped_keys: frozenset[str] = frozenset()
@dataclass(frozen=True, slots=True)
class _InvalidSupplementalParams:
message: str
param: str
def _parse_supplemental_provider_params(
kwargs: Mapping[str, object],
) -> _SupplementalProviderParams | _InvalidSupplementalParams:
supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs}
try:
return _SupplementalProviderParams(params=validate_litellm_params_strictly(GenericLiteLLMParams, supplemental))
except ValidationError as error:
invalid: Final = tuple(detail for detail in error.errors() if detail["loc"])
invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid)
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
return _SupplementalProviderParams(
params=GenericLiteLLMParams.model_validate(
{key: value for key, value in supplemental.items() if key not in invalid_keys}
),
dropped_keys=invalid_keys,
)
return _InvalidSupplementalParams(
message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid),
param=", ".join(sorted(invalid_keys)),
)
def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams:
parsed: Final = _parse_supplemental_provider_params(kwargs)
if isinstance(parsed, _InvalidSupplementalParams):
raise litellm.BadRequestError(
message=parsed.message, model=model, llm_provider=None, body={"param": parsed.param}
)
return parsed
PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
@ -830,21 +874,17 @@ async def acompletion(
"enable_json_schema_validation": enable_json_schema_validation,
}
if custom_llm_provider is None:
_supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects
"dict[str, object]",
{
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
},
)
request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs
_, custom_llm_provider, _, _ = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=cast( # cast-ok: api_base arrives as a str in the request kwargs
"str | None", kwargs.get("api_base")
"str | None", request_kwargs.get("api_base")
)
or base_url,
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
litellm_params=_supplemental_provider_params(
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
).params,
)
if fallbacks is not None:
@ -1197,6 +1237,7 @@ def mock_completion(
custom_llm_provider="openai",
logging_obj=logging,
count_prompt_tokens=count_prompt_tokens,
is_mock=True,
)
return CustomStreamWrapper(
completion_stream=mock_completion_streaming_obj(
@ -1206,6 +1247,7 @@ def mock_completion(
custom_llm_provider="openai",
logging_obj=logging,
count_prompt_tokens=count_prompt_tokens,
is_mock=True,
)
if isinstance(mock_response, litellm.MockException):
raise mock_response
@ -4382,6 +4424,7 @@ _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
@ -5625,6 +5668,11 @@ def completion(
######### unpacking kwargs #####################
args: Final = _locals_snapshot(locals())
supplemental_params: Final = _supplemental_provider_params(
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
)
for dropped_key in supplemental_params.dropped_keys:
kwargs.pop(dropped_key)
# Set by the responses->completion fallback so completion() does not bridge
# back to the Responses API: that round-trip mutually recurses forever for a
@ -5850,21 +5898,14 @@ def completion(
if deployment_id is not None: # azure llms
model = deployment_id
custom_llm_provider = "azure"
_supplemental_provider_params: Final = {
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
}
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
requested_model: Final = model
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
litellm_params=(
GenericLiteLLMParams.model_validate(_supplemental_provider_params)
if _supplemental_provider_params
else None
),
litellm_params=supplemental_params.params,
)
_record_agentic_loop_params(
logging_obj=logging,
@ -5884,7 +5925,6 @@ def completion(
api_base=api_base,
deployment_model_info=deployment_model_info,
)
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
drop_unsupported_reasoning: Final = (
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
)
@ -6220,6 +6260,7 @@ def completion(
"github_copilot_user_session",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_forward_user_id",
"thought_signature_in_tool_call_id",
"responses_api_version",
"allowed_openai_params",
)

View file

@ -81372,5 +81372,77 @@
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-6-astra-law": {
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "openai",
"max_input_tokens": 922000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"reasoning_contexts": [
"auto",
"current_turn",
"all_turns"
],
"reasoning_modes": [
"standard",
"pro"
],
"regional_processing_uplift_multiplier_eu": 1.1,
"regional_processing_uplift_multiplier_us": 1.1,
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,
"supports_none_reasoning_effort": false,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_search": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"chatgpt/gpt-6-astra-law": {
"litellm_provider": "chatgpt",
"max_input_tokens": 922000,
"max_output_tokens": 128000,
"mode": "responses",
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_parallel_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
}
}

View file

@ -24268,7 +24268,14 @@
"properties": {
"content": {
"items": {
"$ref": "#/components/schemas/OutputText"
"anyOf": [
{
"$ref": "#/components/schemas/OutputText"
},
{
"$ref": "#/components/schemas/ResponseOutputRefusal"
}
]
},
"title": "Content",
"type": "array"

View file

@ -38,12 +38,14 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_no_callback_env_reference,
)
from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from litellm.types.agents import (
AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export
)
from litellm.types.agents import (
AgentResponse as AgentResponse, # noqa: PLC0414 # public re-export
)
from litellm.types.decisions import OpenAIDecisionResponse
from litellm.types.integrations.compression_interception import (
CompressionSavingsMetadata,
)
@ -2470,6 +2472,22 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
Documents all the fields supported by `general_settings` in config.yaml
"""
accounting_capture: UsageCaptureSettings | None = Field(
None,
description="Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes.",
)
accounting_sync_ack: bool = Field(
False,
description="Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. "
"Other requests remain supported with asynchronous usage capture.",
)
accounting_sync_ack_timeout_seconds: float = Field(
20.0, gt=0, description="Maximum response wait for usage commit, including the batch flush interval"
)
accounting_sync_ack_failure_policy: Literal["fail_closed", "log"] = Field(
"fail_closed", description="Return 503 or log the persistence failure after the response wait expires"
)
request_tag_mappings: Sequence[RequestTagMapping] | None = Field(
None,
description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.",
@ -4584,6 +4602,7 @@ PassThroughEndpointLoggingResultValues = (
| StandardPassThroughResponseObject
| ResponsesAPIResponse
| TranscriptionResponse
| OpenAIDecisionResponse
)

View file

@ -11,7 +11,7 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
"""
import json
from collections.abc import AsyncGenerator, Mapping
from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol
@ -19,7 +19,8 @@ from urllib.parse import urlparse
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import TypeAdapter, ValidationError
from fastapi.routing import APIRoute
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_proxy_logger
@ -27,7 +28,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.a2a.version_convert import (
A2AVersion,
normalize_agent_card,
@ -42,6 +43,7 @@ from litellm.proxy.agent_endpoints.databricks_oauth import (
)
from litellm.proxy.agent_endpoints.utils import merge_agent_headers
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING,
coerce_keepalive_interval,
@ -172,13 +174,16 @@ def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str,
async def _resolve_backend_auth(
litellm_params: dict[str, object],
custom_llm_provider: object,
agent_extra_headers: Mapping[str, str] | None = None,
) -> AgentBackendAuth:
databricks_auth: Final = await resolve_databricks_backend_auth(litellm_params)
if has_databricks_oauth(litellm_params):
return databricks_auth
return AgentBackendAuth(
litellm_params=databricks_auth.litellm_params,
header=await resolve_a2a_hop_auth_header(litellm_params, custom_llm_provider),
header=await resolve_a2a_hop_auth_header(
litellm_params, custom_llm_provider, agent_extra_headers=agent_extra_headers
),
)
@ -218,6 +223,57 @@ def _jsonrpc_error(
)
class _JsonRpcRequestId(BaseModel):
model_config = ConfigDict(frozen=True)
id: str | int | None = None
async def _jsonrpc_request_id(request: Request) -> str | int | None:
try:
return _JsonRpcRequestId.model_validate_json(await request.body()).id
except ValidationError:
return None
async def _jsonrpc_refusal(request: Request, refusal: ProxyException | HTTPException) -> JSONResponse:
from litellm.proxy.proxy_server import (
_close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; a route that answers an error itself must stamp the OTel server span like the global ProxyException handler does
)
error: Final = refusal if isinstance(refusal, ProxyException) else status_typed_proxy_exception(refusal)
_close_dangling_otel_server_span(request, int(error.code), exc=refusal)
return JSONResponse(
content={
"jsonrpc": "2.0",
"id": await _jsonrpc_request_id(request),
"error": {
"code": -32000,
"message": error.message,
"data": {"type": error.type, "param": error.param, "code": error.code},
},
},
status_code=int(error.code),
headers=error.headers if isinstance(refusal, ProxyException) else refusal.headers,
)
class _JsonRpcRefusalRoute(APIRoute):
def get_route_handler(self) -> Callable[[Request], Coroutine[object, object, Response]]:
handler: Final = super().get_route_handler()
async def answer_refusals_as_jsonrpc(request: Request) -> Response:
try:
return await handler(request)
except (ProxyException, HTTPException) as refusal:
return await _jsonrpc_refusal(request, refusal)
return answer_refusals_as_jsonrpc
jsonrpc_router: Final = APIRouter(route_class=_JsonRpcRefusalRoute)
async def _get_agent(agent_id: str) -> "AgentResponse | None":
"""Look up an agent by ID or name. Returns None if not found."""
from litellm.proxy.common_utils.registry_read_through import (
@ -646,17 +702,17 @@ async def get_agent_card(
raise HTTPException(status_code=500, detail=str(e))
@router.post(
@jsonrpc_router.post(
"/a2a/{agent_id}",
tags=["[beta] A2A Agents"],
dependencies=[Depends(user_api_key_auth)],
)
@router.post(
@jsonrpc_router.post(
"/a2a/{agent_id}/message/send",
tags=["[beta] A2A Agents"],
dependencies=[Depends(user_api_key_auth)],
)
@router.post(
@jsonrpc_router.post(
"/v1/a2a/{agent_id}/message/send",
tags=["[beta] A2A Agents"],
dependencies=[Depends(user_api_key_auth)],
@ -834,14 +890,15 @@ async def invoke_agent_a2a(
if header_name:
dynamic_headers[header_name] = val
backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider)
merged_agent_headers: Final = merge_agent_headers(
dynamic_headers=dynamic_headers or None,
static_headers=static_headers or None,
)
backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider, merged_agent_headers)
agent_extra_headers: Final = _forwarding_headers(
caller_identity=caller_identity,
request_data=data,
agent_extra_headers=merge_agent_headers(
dynamic_headers=dynamic_headers or None,
static_headers=static_headers or None,
),
agent_extra_headers=merged_agent_headers,
backend_auth_header=backend_auth.header,
)
forwarded_litellm_params: Final = backend_auth.litellm_params
@ -1051,3 +1108,6 @@ async def invoke_agent_a2a(
if isinstance(e, litellm.BadRequestError):
return _jsonrpc_error(body.get("id"), -32602, e.message, 400)
return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500)
router.include_router(jsonrpc_router)

View file

@ -1086,13 +1086,31 @@ def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] |
def route_skips_budget_checks(route: str) -> bool:
return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)
if route in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES:
return False
if route in MODEL_DISCOVERY_ROUTES:
return True
if RouteChecks.is_llm_api_route(route=route):
return False
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass-through endpoints import the auth stack
InitPassThroughEndpointHelpers,
)
return not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route)
def request_skips_budget_checks(route: str, model: str | list[str] | None, llm_router: Router | None) -> bool:
return route_skips_budget_checks(route=route) or is_model_cost_zero(model=model, llm_router=llm_router)
def request_skips_budget_checks(
route: str,
model: str | list[str] | None,
llm_router: Router | None,
*,
request_body: Mapping[str, object] | None = None,
) -> bool:
if route_skips_budget_checks(route=route):
return True
if request_body is not None and is_mcp_discovery_request(route=route, request_body=request_body):
return True
return is_model_cost_zero(model=model, llm_router=llm_router)
async def common_checks(
@ -1142,10 +1160,11 @@ async def common_checks(
team_id=valid_token.team_id if valid_token is not None else None,
)
skip_all_budget_checks: Final = (
skip_budget_checks
or route_skips_budget_checks(route=route)
or is_mcp_discovery_request(route=route, request_body=_typed_request_body(request_body))
skip_all_budget_checks: Final = skip_budget_checks or request_skips_budget_checks(
route=route,
model=_model,
llm_router=llm_router,
request_body=_typed_request_body(request_body),
)
membership_user_id: Final = (

View file

@ -80,7 +80,6 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
get_team_object,
get_user_object,
get_user_role,
is_mcp_discovery_request,
is_model_cost_zero,
is_user_proxy_admin,
is_valid_fallback_model,
@ -88,6 +87,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
key_access_group_grants_model,
key_model_aliases_for_auth_check,
model_max_budget_of,
request_skips_budget_checks,
resolve_and_validate_end_user_id,
resolve_default_end_user_budget,
route_skips_budget_checks,
@ -2497,15 +2497,9 @@ async def user_api_key_auth_builder(
llm_router=llm_router,
team_id=valid_token.team_id,
)
skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
if not skip_budget_checks and model is not None and llm_router is not None:
from litellm.proxy.auth.auth_checks import is_model_cost_zero
skip_budget_checks = ( # rebind-ok: pre-existing rebinding on a rename-only line
is_model_cost_zero(model=model, llm_router=llm_router)
)
if skip_budget_checks:
verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
skip_budget_checks = request_skips_budget_checks(
route=route, model=model, llm_router=llm_router, request_body=request_data
)
# Fetch project object for JWT path if project_id is set
_jwt_project_obj = None
@ -3019,15 +3013,9 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
llm_router=llm_router,
team_id=valid_token.team_id,
)
skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
if not skip_budget_checks and model is not None and llm_router is not None:
from litellm.proxy.auth.auth_checks import is_model_cost_zero
skip_budget_checks = is_model_cost_zero( # rebind-ok: pre-existing rebinding on a rename-only line
model=model, llm_router=llm_router
)
if skip_budget_checks:
verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
skip_budget_checks: Final = request_skips_budget_checks(
route=route, model=model, llm_router=llm_router, request_body=request_data
)
# Check 3. Check if user is in their team budget
if not skip_budget_checks and valid_token.team_member_spend is not None:
@ -4088,11 +4076,7 @@ def _should_skip_budget_checks(
llm_router=llm_router,
team_id=team_id,
)
if is_mcp_discovery_request(route=route, request_body=request_data):
return True
if model is not None and llm_router is not None:
return is_model_cost_zero(model=model, llm_router=llm_router)
return False
return request_skips_budget_checks(route=route, model=model, llm_router=llm_router, request_body=request_data)
def _resolve_request_principal(request: Request, valid_token: UserAPIKeyAuth) -> Principal:

View file

@ -137,6 +137,7 @@ from litellm.proxy.route_llm_request import (
route_request,
)
from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost, prepare_response_cost
from litellm.proxy.spend_tracking.usage_capture import capture_proxy_response
from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
ProxyLogging,
@ -3366,6 +3367,9 @@ class ProxyBaseLLMRequestProcessing:
)
),
)
await capture_proxy_response(
logging_obj, response, background=_REQUEST_FIELDS.validate_python(self.data).get("background") is True
)
except (Exception, asyncio.CancelledError) as exc:
_exception_raised = True
if logging_obj.prepared_cost is not None:

View file

@ -275,7 +275,7 @@ class AccountingLedger:
)
return tuple(
UsageWrite(
OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
OutboundUsageEvent.model_validate_json(_json(_JSON.validate_python(row["payload"])["event"])),
_json(row["payload"]),
TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
)

View file

@ -12,18 +12,31 @@ from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
from litellm.llms.anthropic.messages.stream_assembly import (
anthropic_message_delta_fields,
anthropic_sse_error_events,
anthropic_sse_event_type,
anthropic_sse_event_types,
anthropic_sse_started_blocks,
anthropic_sse_stopped_block_index,
anthropic_sse_unfinished_blocks,
)
from litellm.llms.anthropic.messages.stream_assembly import (
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
has_unreadable_json_event as has_unreadable_json_event, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.utils import Choices, ModelResponse
@ -101,6 +114,48 @@ def _rewritten_message_delta_fields(
)
_MESSAGE_END_EVENT_TYPES: Final = frozenset({"message_delta", "message_stop"})
def _text_block_left_open(rewritten_chunks: Sequence[bytes], source_chunks: Sequence[object]) -> int | None:
"""Index of the rewritten text block to leave open when the failed upstream never stopped its text.
Assembly keeps a cut-off text block and drops every other cut-off block, and the round trip
merges all text into one block, so that block carries text the upstream left open. It stays
open only when it is the last block, since a client must see each block stop before the next
one starts.
"""
upstream_left_text_open: Final = any(
block.get("type") == "text" for block in anthropic_sse_unfinished_blocks(source_chunks).values()
)
rewritten_blocks: Final = anthropic_sse_started_blocks(rewritten_chunks)
if not upstream_left_text_open or not rewritten_blocks:
return None
last_index: Final = max(rewritten_blocks)
return last_index if rewritten_blocks[last_index].get("type") == "text" else None
def _is_empty_text(block: object) -> bool:
fields: Final = as_str_mapping(block)
return fields is not None and fields.get("type") == "text" and not fields.get("text")
def _without_empty_text(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
return {**response, "content": [block for block in response.get("content") or () if not _is_empty_text(block)]}
def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[str]) -> bool:
event: Final = chunk.decode()
event_type: Final = anthropic_sse_event_type(event)
if event_type in _MESSAGE_END_EVENT_TYPES:
return event_type in end_events_sent
return block_left_open is None or anthropic_sse_stopped_block_index(event) != block_left_open
def _sse_chunks_of_events(events: Sequence[str]) -> tuple[bytes, ...]:
return tuple(f"{event}\n\n".encode() for event in events)
def anthropic_sse_chunks_from_response(
assembled: ModelResponse, source_chunks: Sequence[object] = (), *, keep_explanations: bool = False
) -> tuple[bytes, ...]:
@ -110,7 +165,26 @@ def anthropic_sse_chunks_from_response(
nothing the OpenAI round trip made up, with a verdict kept only for a tool call delivered with
the input the classifier judged. Their free-text ``explanation`` was written about the text the
guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``).
A source stream that ended in an upstream ``error`` event ends with that event again, never
with a ``message_delta`` or ``message_stop`` the upstream did not send, with its cut-off text
block left open unless another block follows it, and with no finished text block when it read
no text, since Claude Code does not retry a stream that shows a finished block. A response that
cannot be re-emitted, such as one with a tool input that cannot be parsed, leaves a failed stream
its error alone.
"""
error_events: Final = anthropic_sse_error_events(source_chunks)
try:
return _rebuilt_sse_chunks(assembled, source_chunks, error_events, keep_explanations)
except Exception:
if not error_events:
raise
verbose_proxy_logger.warning("Failed Anthropic stream could not be rebuilt, sending its upstream error alone")
return _sse_chunks_of_events(error_events)
def _rebuilt_sse_chunks(
assembled: ModelResponse, source_chunks: Sequence[object], error_events: Sequence[str], keep_explanations: bool
) -> tuple[bytes, ...]:
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
@ -120,7 +194,7 @@ def anthropic_sse_chunks_from_response(
adapter: Final = LiteLLMAnthropicMessagesAdapter()
anthropic_response: Final = adapter.translate_openai_response_to_anthropic(response=assembled)
source_response: Final = assemble_anthropic_sse_stream(source_chunks)
source_response: Final = assemble_anthropic_sse_stream(source_chunks, through_upstream_error=True)
source_content: Final = (
adapter.translate_openai_response_to_anthropic(response=source_response).get("content")
if source_response is not None
@ -129,11 +203,15 @@ def anthropic_sse_chunks_from_response(
unchanged_tool_use_ids: Final = _unchanged_tool_use_ids(
source_content or (), anthropic_response.get("content") or ()
)
return tuple(
FakeAnthropicMessagesStreamIterator(
response=anthropic_response,
message_delta_fields=_rewritten_message_delta_fields(
source_chunks, unchanged_tool_use_ids, keep_explanations
),
).chunks
rewritten_chunks: Final = FakeAnthropicMessagesStreamIterator(
response=_without_empty_text(anthropic_response) if error_events else anthropic_response,
message_delta_fields=_rewritten_message_delta_fields(source_chunks, unchanged_tool_use_ids, keep_explanations),
).chunks
if not error_events:
return tuple(rewritten_chunks)
block_left_open: Final = _text_block_left_open(rewritten_chunks, source_chunks)
end_events_sent: Final = _MESSAGE_END_EVENT_TYPES & anthropic_sse_event_types(source_chunks)
return (
*(chunk for chunk in rewritten_chunks if _sent_by_the_failed_upstream(chunk, block_left_open, end_events_sent)),
*_sse_chunks_of_events(error_events),
)

View file

@ -48,6 +48,7 @@ from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_response,
assemble_anthropic_sse_stream,
has_unreadable_json_event,
is_anthropic_sse_stream,
model_response_text,
)
@ -117,7 +118,7 @@ _MAX_FIRST_SSE_FRAME_BYTES: Final = 64 * 1024
@dataclass(frozen=True, slots=True)
class _SsePreface:
"""Complete leading SSE frames with no ``data:`` line, relayed verbatim before the stream shape is decided."""
"""Complete leading SSE frames that carry no data, relayed verbatim before the stream shape is decided."""
raw: bytes
@ -125,17 +126,27 @@ class _SsePreface:
_SSE_FRAME_END: Final = re.compile(rb"\r\n\r\n|\n\n|\r\r")
def _carries_data(frame: bytes) -> bool:
return any(line.startswith(b"data:") and line[len(b"data:") :].strip() for line in frame.splitlines())
def _split_sse_preface(complete_frames: bytes) -> tuple[bytes, bytes]:
"""Split complete frames into ``(frames before the first data-bearing frame, that frame and everything after)``."""
start = 0
for end in _SSE_FRAME_END.finditer(complete_frames):
frame = complete_frames[start : end.end()]
if any(line.startswith(b"data:") for line in frame.splitlines()):
if _carries_data(frame):
return complete_frames[:start], complete_frames[start:]
start = end.end()
return complete_frames, b""
def _holds_json_object_frame(frames: bytes) -> bool:
return any(
line.startswith(b"data:") and line[len(b"data:") :].lstrip().startswith(b"{") for line in frames.splitlines()
)
def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[ModelResponseStream]:
"""Buffered chunks flushed unmasked when a mixed stream shape makes reconstruction impossible."""
if not all_chunks:
@ -150,12 +161,13 @@ def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[
async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGenerator[object, None]:
"""
Relay leading data-less SSE frames (comment keepalives, events without a
``data:`` line) as they complete, and join raw ``bytes`` chunks until they
hold one complete SSE event with a data line, so the stream shape is
decided on a whole frame rather than a transport fragment. Everything
after that first frame is forwarded untouched. The byte cap can only be
reached by a single unterminated frame.
Relay leading data-less SSE frames (comment keepalives, events with no
data) as they complete, and join raw ``bytes`` chunks until they
hold one complete SSE event with a JSON object on its data line, so the
stream shape is decided on a whole frame rather than a transport fragment
or a line that says nothing about the shape. Everything after that frame
is forwarded untouched. The byte cap bounds a single unterminated frame
and a run of frames with no JSON object.
"""
pending = b""
try:
@ -169,7 +181,7 @@ async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGener
if preface:
yield _SsePreface(preface)
pending = classifiable + tail
if classifiable or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
if _holds_json_object_frame(classifiable) or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
break
else:
if pending:
@ -1523,7 +1535,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
first_frame_is_anthropic = (
not passthrough_due_to_unknown_stream_shape
and not all_chunks
and is_anthropic_sse_stream((chunk,))
and self._is_anthropic_first_frame(chunk)
)
if not first_frame_is_anthropic:
passthrough_due_to_unknown_stream_shape = (
@ -1565,13 +1577,29 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
for chunk in all_chunks:
yield chunk
def _is_anthropic_first_frame(self, chunk: bytes) -> bool:
if is_anthropic_sse_stream((chunk,)):
return True
if has_unreadable_json_event((chunk,)):
raise self._unreadable_stream_refusal()
return False
def _unreadable_stream_refusal(self) -> GuardrailRaisedException:
return GuardrailRaisedException(
guardrail_name=self.guardrail_name,
message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured",
should_wrap_with_default_message=False,
)
async def _mask_anthropic_sse_stream(
self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict
) -> tuple[object, ...]:
rest_chunks: Final = [chunk async for chunk in rest]
chunks: Final = (first_chunk, *rest_chunks)
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True)
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True)
if assembled is None:
if has_unreadable_json_event(chunks):
raise self._unreadable_stream_refusal()
verbose_proxy_logger.warning(
"Presidio apply_to_output: raw SSE stream could not be assembled into a response. "
"Output PII masking was skipped for this response."

View file

@ -25,8 +25,10 @@ from litellm.proxy.common_utils.callback_utils import (
)
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_response,
anthropic_sse_error_events,
assemble_anthropic_sse_stream,
is_raw_sse_stream,
is_sse_error_stream,
)
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
@ -333,7 +335,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
parsed_arguments = arguments
else:
return None, "arguments must be a JSON object"
except (json.JSONDecodeError, TypeError) as exc:
except (json.JSONDecodeError, TypeError, RecursionError) as exc:
verbose_proxy_logger.warning(
"Tool Permission Guardrail: Failed to decode arguments for tool %s: %s",
tool_call.function.name,
@ -957,9 +959,11 @@ class ToolPermissionGuardrail(CustomGuardrail):
yield chunk
return
anthropic_response: Final = assemble_anthropic_sse_stream(all_chunks)
anthropic_response: Final = assemble_anthropic_sse_stream(
all_chunks, restore_identity=bool(anthropic_sse_error_events(all_chunks)), through_upstream_error=True
)
if anthropic_response is None:
if is_raw_sse_stream(all_chunks):
if is_raw_sse_stream(all_chunks) and not is_sse_error_stream(all_chunks):
raise GuardrailRaisedException(
guardrail_name=self.guardrail_name,
message=(
@ -971,16 +975,24 @@ class ToolPermissionGuardrail(CustomGuardrail):
yield chunk
return
anthropic_denials: Final = self._check_assembled_stream(anthropic_response)
if not anthropic_denials:
for chunk in all_chunks:
yield chunk
return
self._modify_response_with_permission_errors(anthropic_response, anthropic_denials)
for sse_chunk in anthropic_sse_chunks_from_response(anthropic_response, all_chunks, keep_explanations=True):
for sse_chunk in self._anthropic_stream_to_deliver(anthropic_response, all_chunks):
yield sse_chunk
def _anthropic_stream_to_deliver(
self, anthropic_response: ModelResponse, source_chunks: Sequence[object]
) -> Sequence[object]:
"""The frames the client gets: on a failed upstream only what the check read, never the upstream's own.
Assembly reads past an upstream error, and a client can read a frame of a broken stream that assembly
does not, so forwarding those frames would pass a tool call nobody checked.
"""
denials: Final = self._check_assembled_stream(anthropic_response)
if denials:
self._modify_response_with_permission_errors(anthropic_response, denials)
elif not anthropic_sse_error_events(source_chunks):
return source_chunks
return anthropic_sse_chunks_from_response(anthropic_response, source_chunks, keep_explanations=True)
def _check_assembled_stream(
self, assembled: ModelResponse
) -> tuple[tuple[ChatCompletionMessageToolCall, PermissionError], ...]:

View file

@ -21,7 +21,7 @@ from litellm.proxy.auth.auth_utils import (
)
from litellm.proxy.auth.budget_throttle import throttled_limit
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.types.utils import CallTypes, Usage
@ -41,10 +41,12 @@ def _precise_minute(now: datetime) -> str:
return now.strftime("%Y-%m-%d-%H-%M")
def _response_total_tokens(response_obj: object) -> int:
if not isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)):
return 0
response_usage: Final = getattr(response_obj, "usage", None)
def _response_total_tokens(kwargs: Mapping[str, object], response_obj: object) -> int:
response_usage: Final = (
getattr(response_obj, "usage", None)
if isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse))
else socket_session_usage(kwargs, response_obj)
)
return response_usage.total_tokens if isinstance(response_usage, Usage) else 0
@ -556,7 +558,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
precise_minute: Final = _precise_minute(self._clock())
total_tokens: int = _response_total_tokens(response_obj)
total_tokens: Final = _response_total_tokens(
kwargs, # pyright: ignore[reportUnknownArgumentType] # legacy callback kwargs are untyped
response_obj,
)
# ------------
# Update usage - API Key
@ -630,8 +635,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - User
# ------------
if user_api_key_user_id is not None:
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_user_id}::{precise_minute}::request_count"
current = await self.internal_usage_cache.async_get_cache(
@ -639,7 +642,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
litellm_parent_otel_span=litellm_parent_otel_span,
) or {
"current_requests": 1,
"current_tpm": total_tokens,
"current_tpm": 0,
"current_rpm": 1,
}
@ -656,8 +659,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - Team
# ------------
if user_api_key_team_id is not None:
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_team_id}::{precise_minute}::request_count"
current = await self.internal_usage_cache.async_get_cache(
@ -665,7 +666,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
litellm_parent_otel_span=litellm_parent_otel_span,
) or {
"current_requests": 1,
"current_tpm": total_tokens,
"current_tpm": 0,
"current_rpm": 1,
}
@ -682,8 +683,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - End User
# ------------
if user_api_key_end_user_id is not None:
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_end_user_id}::{precise_minute}::request_count"
current = await self.internal_usage_cache.async_get_cache(
@ -691,7 +690,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
litellm_parent_otel_span=litellm_parent_otel_span,
) or {
"current_requests": 1,
"current_tpm": total_tokens,
"current_tpm": 0,
"current_rpm": 1,
}

View file

@ -46,8 +46,10 @@ from litellm.constants import (
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
)
from litellm.cost_calculator import get_usage_object
from litellm.exceptions import ServiceUnavailableError
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
@ -85,7 +87,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
batch_response_view,
canonical_provider_batch_id,
)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
from litellm.proxy.spend_tracking.input_tokens import (
inline_pdf_file_count,
models_counted_for,
@ -745,6 +747,7 @@ class RequestRateLimiterStash:
batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
reservation_released: bool = False
tpm_limited_tags: frozenset[str] = field(default_factory=frozenset)
socket_agent_windows: tuple[RateLimitDescriptor, ...] = ()
@dataclass(frozen=True, slots=True)
@ -864,6 +867,62 @@ def _call_id_from_callback_kwargs(kwargs: object) -> str | None:
return call_id if isinstance(call_id, str) else None
def _socket_connection_call_id(kwargs: Mapping[str, object]) -> object:
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params"))
litellm_metadata: Final = None if litellm_params is None else as_str_mapping(litellm_params.get("litellm_metadata"))
return None if litellm_metadata is None else litellm_metadata.get(RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY)
def _is_socket_turn(kwargs: Mapping[str, object]) -> bool:
if kwargs.get("call_type") == CallTypes.aresponses_websocket.value:
return True
connection_stash: Final = get_request_stash()
connection_call_id: Final = _socket_connection_call_id(kwargs)
return (
connection_stash is not None
and connection_call_id is not None
and connection_call_id == connection_stash.owner_litellm_call_id
)
_AGENT_DESCRIPTOR_KEYS: Final = frozenset({"agent", "agent_session"})
_SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS: Final = frozenset(
{PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, PTU_TEAM_DESCRIPTOR_KEY}
)
def _socket_turn_tpm_windows(descriptors: Sequence[RateLimitDescriptor]) -> tuple[RateLimitDescriptor, ...]:
return tuple(
RateLimitDescriptor(
key=descriptor["key"],
value=descriptor["value"],
rate_limit=RateLimitDescriptorRateLimitObject(
tokens_per_unit=tokens_per_unit, window_size=(descriptor.get("rate_limit") or {}).get("window_size")
),
)
for descriptor in descriptors
if descriptor["key"] not in _SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS
and (tokens_per_unit := (descriptor.get("rate_limit") or {}).get("tokens_per_unit")) is not None
)
def _unmodeled_response_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
combined_usage: Final = kwargs.get("combined_usage_object")
if isinstance(combined_usage, Usage):
return combined_usage
return socket_session_usage(kwargs, response_obj)
def _tpm_limited_tags(call_stash: RequestRateLimiterStash | None, kwargs: Mapping[str, object]) -> frozenset[str]:
if call_stash is not None:
return call_stash.tpm_limited_tags
connection_stash: Final = get_request_stash()
if connection_stash is None or not _is_socket_turn(kwargs):
return frozenset()
return connection_stash.tpm_limited_tags
def _parse_output_cap_value(raw_value: object) -> int | None:
if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float, str)):
return None
@ -2967,6 +3026,40 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if response["overall_code"] == "OVER_LIMIT":
self._handle_rate_limit_error(response, descriptors, requested_model)
@with_service_target("rate_limits")
async def enforce_token_limits_for_frame(
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
) -> None:
connection_stash: Final = get_request_stash()
tags: Final = sorted(connection_stash.tpm_limited_tags) if connection_stash is not None else []
agent_windows: Final = connection_stash.socket_agent_windows if connection_stash is not None else ()
hidden_stash: Final = _request_stash.set(None)
try:
descriptors: Final = await self._build_request_rate_limit_descriptors(
user_api_key_dict,
MappingProxyType({"model": requested_model, "metadata": {"tags": tags}}),
CallTypes.aresponses_websocket.value,
)
finally:
_request_stash.reset(hidden_stash)
built_scopes: Final = frozenset((d["key"], d["value"]) for d in descriptors)
tpm_windows: Final = _socket_turn_tpm_windows(
(*descriptors, *(d for d in agent_windows if (d["key"], d["value"]) not in built_scopes))
)
try:
response: Final = await self.should_rate_limit(
tpm_windows,
parent_otel_span=user_api_key_dict.parent_otel_span,
read_only=True,
skip_parallel_check=True,
)
except RateLimitUnverifiableError as unverifiable:
raise ServiceUnavailableError(
message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model or ""
) from unverifiable
if response["overall_code"] == "OVER_LIMIT":
self._handle_rate_limit_error(response, tpm_windows, requested_model)
def _rate_limited_model(self, requested_model: str | None) -> RateLimitedModel | None:
if not requested_model:
return None
@ -3777,7 +3870,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
def _handle_rate_limit_error(
self,
response: RateLimitResponse,
descriptors: list[RateLimitDescriptor],
descriptors: Sequence[RateLimitDescriptor],
requested_model: str | None = None,
) -> None:
"""Handle rate limit exceeded by raising :class:`ProxyRateLimitError` (a 429)."""
@ -4329,6 +4422,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for d in descriptors
if d["key"] == "tag" and d["rate_limit"] is not None and d["rate_limit"].get("tokens_per_unit") is not None
)
stash.socket_agent_windows = tuple(d for d in descriptors if d["key"] in _AGENT_DESCRIPTOR_KEYS)
# Only check rate limits if we have descriptors with actual limits
if descriptors:
@ -5286,9 +5380,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# Get total tokens from response. Responses LiteLLM does not model
# (e.g. pass-through, whose usage is reported by the upstream rather
# than parsed out of the body) carry their usage in
# ``combined_usage_object`` instead, and would otherwise never charge
# the TPM window.
_combined_usage: Final = kwargs.get("combined_usage_object")
# ``combined_usage_object`` instead (a Responses WebSocket turn in its
# raw events) and would otherwise never charge the TPM window.
_usage: Final[Usage | dict[str, object] | None] = (
getattr(response_obj, "usage", None)
if isinstance(
@ -5302,9 +5395,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
else get_usage_object(response_obj)
if isinstance(response_obj, ExtractionResponse)
else _combined_usage
if isinstance(_combined_usage, Usage)
else None
else _unmodeled_response_usage(kwargs, response_obj)
)
total_tokens = self._get_total_tokens_from_usage(usage=_usage, rate_limit_type=rate_limit_type)
if total_tokens == 0:
@ -5337,7 +5428,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
targets: Final = self._collect_tpm_scope_targets(
standard_logging_metadata=standard_logging_metadata,
kwargs=kwargs,
tpm_limited_tags=stash.tpm_limited_tags if stash is not None else frozenset(),
tpm_limited_tags=_tpm_limited_tags(stash, kwargs),
model_group=reconcile_model.group if reconcile_model is not None else None,
)
targets.extend(
@ -5365,16 +5456,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
reserved_tokens=reserved_tokens,
)
)
ptu_skips_socket_usage: Final = kwargs.get("call_type") == CallTypes.aresponses_websocket.value
pipeline_operations.extend(
self._build_team_ptu_tpm_ops(
standard_logging_metadata, # pyright: ignore[reportUnknownArgumentType] # untyped logging metadata
usage_source=response_obj if _usage is None else _usage,
usage_source=response_obj if _usage is None or ptu_skips_socket_usage else _usage,
reconcile_model=reconcile_model,
reserved_scopes=reserved_scopes,
reserved_ceiling=stash.ptu_ceiling if stash is not None else None,
reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0,
reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(),
total_tokens=total_tokens,
total_tokens=0 if ptu_skips_socket_usage else total_tokens,
)
)
@ -5469,27 +5561,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else None,
)
pipeline_operations: Final = self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
),
litellm_parent_otel_span,
)
unguarded_operations: Final = [
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
]
window_guarded_operations: Final = tuple(
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
)
if unguarded_operations:
await self.async_increment_tokens_with_ttl_preservation(
pipeline_operations=unguarded_operations,
parent_otel_span=litellm_parent_otel_span,
)
if window_guarded_operations:
await self.async_increment_reservation_aware_tokens(
pipeline_operations=window_guarded_operations,
parent_otel_span=litellm_parent_otel_span,
)
io_token_operations: Final = self._build_io_token_reservation_ops(
kwargs=kwargs,
response_obj=response_obj,
@ -5509,6 +5588,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
except Exception as e:
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
async def _apply_usage_operations(
self, pipeline_operations: Sequence[RedisPipelineIncrementOperation], parent_otel_span: Span | None
) -> None:
unguarded_operations: Final = [
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
]
window_guarded_operations: Final = tuple(
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
)
if unguarded_operations:
await self.async_increment_tokens_with_ttl_preservation(
pipeline_operations=unguarded_operations,
parent_otel_span=parent_otel_span,
)
if window_guarded_operations:
await self.async_increment_reservation_aware_tokens(
pipeline_operations=window_guarded_operations,
parent_otel_span=parent_otel_span,
)
@with_service_target("rate_limits")
async def async_logging_hook(
self,
@ -5622,11 +5721,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
try:
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(kwargs)
failed_call: Final = _callback_kwargs_mapping(kwargs)
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(failed_call)
pipeline_operations: Final[list[RedisPipelineIncrementOperation]] = []
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(failed_call))
await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span, in_logging_callback=True)
# Skip the reservation refund if async_post_call_failure_hook
@ -5638,7 +5738,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if stash is None or stash.reservation_released
else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
)
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(kwargs)
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call)
if stash is not None and reserved_tokens > 0:
verbose_proxy_logger.debug(
@ -5722,6 +5822,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.reservation_released = True
if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call):
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
kwargs=dict(failed_call), response_obj=None, rate_limit_type=self.get_rate_limit_type()
),
litellm_parent_otel_span,
)
except Exception as e:
verbose_proxy_logger.exception("Error in rate limit failure event: %s", e)

View file

@ -22,7 +22,11 @@ from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
get_parent_otel_span_from_kwargs,
)
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
UsageCaptureObservation,
UsageObservation,
)
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
@ -63,6 +67,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
sanitize_error_information_for_spend_logs,
should_store_prompts_and_responses_in_spend_logs,
)
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs
from litellm.proxy.utils import (
REJECTED_BEFORE_REQUEST_SETUP,
REJECTED_BEFORE_REQUEST_SETUP_KEY,
@ -344,11 +349,27 @@ def _attempt_spend_row_kwargs(
"litellm_params": litellm_params,
"combined_usage_object": attempt.usage,
"response_cost": attempt.response_cost,
"_accounting_usage_observed": UsageObservation.ESTIMATED,
"_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
attempt.usage, UsageObservation.ESTIMATED, attempt.response_cost
),
"_accounting_prior_captures": (),
}
row.pop("billed_failed_attempts", None)
row.pop("_accounting_capture", None)
call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id")
if isinstance(call_id, str):
row["litellm_call_id"] = f"{call_id}{suffix}"
row["parent_request_id"] = call_id
metadata["parent_request_id"] = call_id
captured_attempts: Final = request_data.get("_accounting_billed_attempts")
if isinstance(captured_attempts, tuple):
for captured in TypeAdapter(tuple[object, ...]).validate_python(captured_attempts):
if (
isinstance(captured, CapturedUsage)
and captured.payload["litellm_call_id"] == row["litellm_call_id"]
):
row["_accounting_capture"] = captured
standard_logging_object: Final = request_data.get("standard_logging_object")
if isinstance(standard_logging_object, dict):
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
@ -385,7 +406,7 @@ class _ProxyDBLogger(CustomLogger):
self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime
) -> None:
if self.spend_event_producer is None or not is_offloadable_success(response_obj):
await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time)
await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time)
return
event: Final = build_spend_event(
kwargs,
@ -396,7 +417,7 @@ class _ProxyDBLogger(CustomLogger):
)
if isinstance(event, SpendEventBuildError):
verbose_proxy_logger.warning("collector: tracking cost in-process, event not buildable: %s", event.reason)
await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time)
await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time)
return
await self.spend_event_producer.publish(event)
@ -434,6 +455,9 @@ class _ProxyDBLogger(CustomLogger):
"Failed to invalidate budget reservation counters after failure release failed"
)
capture_metadata: Final = await self._try_capture_failure_usage(
request_data, original_exception, user_api_key_dict
)
if completed_cost and tracking_disabled:
return
@ -477,8 +501,12 @@ class _ProxyDBLogger(CustomLogger):
)
_metadata["error_information"] = _error_information
_metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( # rebind-ok: pre-existing rebinding on a rename-only line
metadata=_metadata, original_exception=original_exception
_metadata = ( # rebind-ok: enriched identity is shared with the earlier immutable capture
{**_metadata, **capture_metadata}
if capture_metadata is not None
else await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled(
metadata=_metadata, original_exception=original_exception
)
)
request_fields: Final = _str_keyed_fields(request_data)
@ -595,6 +623,106 @@ class _ProxyDBLogger(CustomLogger):
),
)
async def _failure_capture_metadata(
self, request_data: Mapping[str, object], exception: Exception, auth: UserAPIKeyAuth
) -> Mapping[str, object] | None:
from litellm.proxy.proxy_server import usage_capture
if usage_capture.meter is None:
return None
captured_identity: Final = (
_batch_accounting_for(request_data.get("litellm_call_id")) is not None
and request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is not REJECTED_BEFORE_REQUEST_SETUP
)
current_identity: Final = {
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
"user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
}
metadata: Final = (
{
**dict.fromkeys(current_identity),
**{
key: value
for key, value in _REQUEST_DATA.validate_python(
get_litellm_metadata_from_kwargs(dict(request_data))
).items()
if key in current_identity or key == "tags"
},
}
if captured_identity
else current_identity
)
return await self._enrich_failure_metadata_unless_db_stalled(
metadata=metadata, original_exception=exception, resolve_missing_key_identity=not captured_identity
)
async def _try_capture_failure_usage(
self,
request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer
original_exception: Exception,
auth: UserAPIKeyAuth,
) -> Mapping[str, object] | None:
try:
metadata: Final = await self._failure_capture_metadata(request_data, original_exception, auth)
self._capture_failure_usage(request_data, original_exception, auth, metadata)
except Exception: # noqa: BLE001 # capture failure must not suppress legacy billing or reservation release
verbose_proxy_logger.exception("Usage capture failed on the request failure path")
return None
return metadata
def _capture_failure_usage(
self,
request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer
original_exception: Exception,
auth: UserAPIKeyAuth,
metadata: Mapping[str, object] | None = None,
) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.db.exception_handler import exception_chain
from litellm.proxy.proxy_server import usage_capture
from litellm.proxy.spend_tracking.usage_capture import UsageAcknowledgementError
if any(isinstance(error, UsageAcknowledgementError) for error in exception_chain(original_exception)):
return
usage_meter: Final = usage_capture.meter
if usage_meter is None:
return
logging_obj: Final = request_data.get("litellm_logging_obj")
details: Final[Mapping[str, object]] = (
_str_keyed_fields(logging_obj.model_call_details) if isinstance(logging_obj, Logging) else {}
)
params: Final = _str_keyed_fields(request_data.get("litellm_params") or details.get("litellm_params"))
failure_data: Final = {
**details,
**request_data,
"litellm_params": {
**params,
"metadata": {
**_str_keyed_fields(params.get("metadata")),
**(
metadata
if metadata is not None
else {
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
"user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
}
),
"status": "failure",
},
},
}
start: Final = logging_obj.start_time if isinstance(logging_obj, Logging) else request_data.get("start_time")
if not isinstance(start, datetime):
return
end: Final = datetime.now(timezone.utc)
captured: Final = tuple(
capture_callback(usage_meter, row, original_exception, start, end)
for _, row in _failure_spend_rows(failure_data, 0.0, None, self._deployment_lookup)
)
request_data["_accounting_capture"] = captured[0] # rebind-ok: share one receipt with the legacy writer
request_data["_accounting_billed_attempts"] = captured[1:] # rebind-ok: share attempt receipts
async def _write_failure_spend_rows(
self,
user_api_key_dict: UserAPIKeyAuth,
@ -780,6 +908,7 @@ class _ProxyDBLogger(CustomLogger):
resolve_missing_key_identity=str(kwargs.get("call_type")) not in _CAPTURED_IDENTITY_CALL_TYPES,
)
_write_spend_metadata_to_kwargs(kwargs=kwargs, metadata=metadata)
kwargs.update(await captured_callback_kwargs(kwargs, completion_response, start_time, end_time))
budget_reservation: Final = _get_budget_reservation_from_metadata(metadata=metadata)
if accounting is not None and (budget_reservation is not None or litellm.apply_search_free_tier):
accounting.write_started = True
@ -967,11 +1096,13 @@ class _ProxyDBLogger(CustomLogger):
@staticmethod
async def _enrich_failure_metadata_unless_db_stalled(
metadata: dict[str, object], original_exception: Exception
metadata: dict[str, object], original_exception: Exception, resolve_missing_key_identity: bool = True
) -> dict[str, object]:
if isinstance(original_exception, DBLookupDeadlineExceeded):
return metadata
return await ProxyDBLogger._enrich_failure_metadata_with_key_info(metadata=metadata)
return await ProxyDBLogger._enrich_failure_metadata_with_key_info(
metadata=metadata, resolve_missing_key_identity=resolve_missing_key_identity
)
@staticmethod
async def _enrich_failure_metadata_with_key_info(metadata: dict, resolve_missing_key_identity: bool = True) -> dict:

View file

@ -2,14 +2,32 @@
Shared utility functions for rate limiter hooks.
"""
from collections.abc import Mapping
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
from litellm.litellm_core_utils.core_helpers import iter_items
from litellm.types.router import ModelGroupInfo
from litellm.types.utils import PriorityReservationDict
from litellm.types.utils import CallTypes, PriorityReservationDict, Usage
_SOCKET_SESSION_EVENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def socket_session_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
if not isinstance(response_obj, list):
return None
match kwargs.get("call_type"):
case CallTypes.aresponses_websocket.value:
return ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
_SOCKET_SESSION_EVENTS.validate_python(response_obj)
)
case _:
return None
def resolve_llm_provider_for_rate_limit(

View file

@ -64,11 +64,16 @@ from litellm.llms.azure.passthrough.transformation import (
from litellm.llms.bedrock.passthrough.transformation import is_bedrock_streaming_endpoint
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.deepgram.common_utils import (
DeepgramSpeakMeter,
deepgram_listen_callback_params,
deepgram_listen_is_priced,
deepgram_listen_registry_key,
deepgram_listen_requested_model,
deepgram_listen_websocket_target,
deepgram_speak_character_rate,
deepgram_speak_registry_key,
deepgram_speak_requested_model,
deepgram_speak_websocket_target,
)
from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
@ -129,6 +134,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
YieldsToConfiguredPassThroughRoute,
create_pass_through_route,
create_websocket_passthrough_route,
truncated_close_reason,
websocket_passthrough_request,
)
from litellm.proxy.utils import ProxyLogging as ProxyLoggingType
@ -145,6 +151,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
WebSocketSessionMeter,
)
from litellm.types.passthrough_endpoints.tinyfish import (
TINYFISH_AUTHENTICATED_RUN_FIELDS,
@ -3460,6 +3467,7 @@ class _WebsocketRelay(Protocol):
forward_headers: bool,
endpoint: str,
accept_websocket: bool,
meter: WebSocketSessionMeter | None = None,
) -> None: ...
@ -3621,6 +3629,54 @@ async def deepgram_listen_websocket_route(
)
async def deepgram_speak_user_api_key_auth(websocket: WebSocket) -> UserAPIKeyAuth:
return await user_api_key_auth_websocket_for_model(
websocket, model=deepgram_speak_requested_model(websocket.url.query)
)
@router.websocket("/deepgram/v1/speak")
@router.websocket("/deepgram/speak")
async def deepgram_speak_websocket_route(
websocket: WebSocket,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(deepgram_speak_user_api_key_auth)],
relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)],
) -> None:
deepgram_api_key: Final = passthrough_endpoint_router.get_credentials(
custom_llm_provider=litellm.LlmProviders.DEEPGRAM.value,
region_name=None,
)
if deepgram_api_key is None:
await websocket.close(code=1011, reason=_DEEPGRAM_WS_MISSING_KEY_REASON)
return
await websocket.accept(subprotocol=_negotiated_websocket_subprotocol(websocket))
target: Final = deepgram_speak_websocket_target(
api_base=get_secret_str("DEEPGRAM_API_BASE"),
query_string=websocket.url.query,
)
character_rate: Final = deepgram_speak_character_rate(target)
if character_rate is None:
await websocket.close(
code=1008,
reason=truncated_close_reason(
_DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_speak_registry_key(target))
),
)
return
await relay(
websocket=websocket,
target=target,
custom_headers={"Authorization": f"Token {deepgram_api_key}"},
user_api_key_dict=user_api_key_dict,
forward_headers=False,
endpoint=websocket.url.path,
accept_websocket=False,
meter=DeepgramSpeakMeter(target, character_rate),
)
_OPENAI_BETA_HEADER: Final = "openai-beta"
_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"})
@ -4474,10 +4530,16 @@ async def watsonx_proxy_route(
if provider_config is None:
raise HTTPException(status_code=404, detail="Watsonx passthrough config not found")
watsonx_credentials: Final = passthrough_endpoint_router.get_proxy_wide_deployment_credentials(
custom_llm_provider="watsonx"
)
api_key: Final = watsonx_credentials.api_key if watsonx_credentials is not None else None
api_base: Final = watsonx_credentials.api_base if watsonx_credentials is not None else None
# Get complete URL with version parameter
complete_url, _ = provider_config.get_complete_url(
api_base=None,
api_key=None,
api_base=api_base,
api_key=api_key,
model="",
endpoint=endpoint,
request_query_params=None,
@ -4491,8 +4553,8 @@ async def watsonx_proxy_route(
messages=[],
optional_params={},
litellm_params={},
api_key=None,
api_base=None,
api_key=api_key,
api_base=api_base,
)
# Check for streaming

View file

@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
from litellm.types.decisions import OpenAIDecisionResponse
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
SETTLES_BACKGROUND_RESPONSE_KWARG,
@ -277,6 +278,22 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
def is_openai_agents_route(url_route: str) -> bool:
return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path)
@staticmethod
def is_openai_decisions_route(url_route: str) -> bool:
return is_openai_compatible_url(url_route) and urlparse(url_route).path.rstrip("/").endswith("/v1/decisions")
@staticmethod
def handles_route(url_route: str) -> bool:
return (
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
)
@staticmethod
def _unbilled_call_model(
url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object]
@ -479,8 +496,19 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
httpx_response.request.method, url_route, response_body
)
is_responses: Final = responses_billing != "unbilled"
is_decisions: Final = (
httpx_response.request.method == "POST"
and OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
)
if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses):
if not (
is_chat_completions
or is_embeddings
or is_image_generation
or is_image_editing
or is_responses
or is_decisions
):
OpenAIPassthroughLoggingHandler._label_unbilled_call(
logging_obj,
url_route,
@ -512,7 +540,13 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
try:
response_cost = 0.0
litellm_model_response: (
ModelResponse | TextCompletionResponse | EmbeddingResponse | ImageResponse | ResponsesAPIResponse | None
ModelResponse
| TextCompletionResponse
| EmbeddingResponse
| ImageResponse
| ResponsesAPIResponse
| OpenAIDecisionResponse
| None
) = None
handler_instance: Final = OpenAIPassthroughLoggingHandler()
@ -564,11 +598,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
response_body=response_body,
request_body=request_body,
)
# Mark call type for downstream image-aware logic/metrics
try:
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
except Exception:
pass
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
# Create a simple response object for logging
litellm_model_response = ImageResponse(
data=response_body.get("data", []),
@ -583,11 +613,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
response_body=response_body,
request_body=request_body,
)
# Mark call type for downstream image-aware logic/metrics
try:
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
except Exception:
pass
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
# Create a simple response object for logging
litellm_model_response = ImageResponse(
data=response_body.get("data", []),
@ -609,6 +635,14 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
)
elif is_decisions:
litellm_model_response = OpenAIDecisionResponse.model_validate(response_body) # rebind-ok: elif-chain
response_cost = litellm.completion_cost( # rebind-ok: elif-chain
completion_response=litellm_model_response,
model=model,
custom_llm_provider=custom_llm_provider,
call_type="decisions",
)
# Update kwargs with cost information
kwargs["response_cost"] = response_cost
@ -656,6 +690,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
if is_image_generation
else "image_editing"
if is_image_editing
else "decisions"
if is_decisions
else "responses"
)
verbose_proxy_logger.debug(

View file

@ -2584,7 +2584,7 @@ def _json_object_frame(frame: str | bytes) -> dict[str, object] | None:
return decoded if isinstance(decoded, dict) else None
def _truncated_close_reason(reason: str) -> str:
def truncated_close_reason(reason: str) -> str:
"""
Fit a close reason inside the byte budget a WebSocket close frame allows, without splitting a character
"""
@ -2958,7 +2958,7 @@ async def websocket_passthrough_request(
if upstream_close is not None and _client_socket_is_open(websocket):
await websocket.close(
code=upstream_close.code,
reason=_truncated_close_reason(upstream_close.reason),
reason=truncated_close_reason(upstream_close.reason),
)
end_time: Final = datetime.now()

View file

@ -132,6 +132,19 @@ class PassthroughEndpointRouter:
api_base=deployment_credentials.api_base if deployment_credentials is not None else None,
)
def get_proxy_wide_deployment_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None:
deployment_credentials: Final = self._get_deployment_credentials(
custom_llm_provider=custom_llm_provider,
region_name=None,
include_team_deployments=False,
)
if deployment_credentials is None:
return None
api_key: Final = _resolve_oidc_reference(deployment_credentials.api_key)
if not api_key:
return None
return _PassThroughCredentials(api_key=api_key, api_base=deployment_credentials.api_base)
def has_credentials(self, custom_llm_provider: str) -> bool:
if (
self._get_deployment_credentials(

View file

@ -666,28 +666,11 @@ class PassThroughEndpointLogging:
return False
def _is_supported_openai_endpoint(self, url_route: str) -> bool:
"""Check if the OpenAI endpoint is supported by the passthrough logging handler.
The Responses API route is included because
`openai_passthrough_handler` has a dedicated `elif is_responses:`
branch that knows how to extract usage + cost from the
Responses-API on-the-wire shape. Without including it here, the
outer dispatch filters Responses calls out before reaching the
handler — the inner branch is then unreachable and Responses
calls land in `LiteLLM_SpendLogs` with zero tokens / zero spend.
"""
from .llm_provider_handlers.openai_passthrough_logging_handler import (
OpenAIPassthroughLoggingHandler,
)
return (
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
)
return OpenAIPassthroughLoggingHandler.handles_route(url_route)
def _set_cost_per_request(
self,

View file

@ -473,7 +473,6 @@ from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # l
log_llm_api_exception,
open_sse_before_first_byte,
proxy_exception_from_route_error,
request_litellm_call_id,
resolve_litellm_call_id,
should_return_raw_model_name,
sse_error_payload,
@ -840,11 +839,15 @@ from litellm.proxy.spend_tracking.spend_event_producer import (
SpendEventProducer,
build_spend_event_producer,
)
from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from litellm.proxy.telemetry.endpoints import router as telemetry_router
from litellm.proxy.telemetry.middleware import TelemetryMiddleware
from litellm.proxy.telemetry.runtime import TelemetryRuntime
from litellm.proxy.telemetry.settings import describe_errors as describe_telemetry_errors
from litellm.proxy.telemetry.settings import load_settings as load_telemetry_settings
from litellm.router import router_arguments
from litellm.router_config import build_router_config
try:
from litellm.proxy.enterprise_billing.billing_metrics import (
@ -1329,6 +1332,7 @@ async def _flush_spend_around_logging_drain_on_shutdown(
finally:
await worker.stop(discard_pending=True, timeout=GracefulShutdownManager.remaining(2.0, clock=clock))
await drain_tasks(GracefulShutdownManager.remaining(5.0, clock=clock))
await usage_capture.stop(GracefulShutdownManager.remaining(clock=clock))
await flush_spend()
@ -3202,9 +3206,39 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False):
spend_event_producer: SpendEventProducer | None = None
usage_capture: Final = UsageCaptureRuntime()
def _configure_usage_capture() -> None:
settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings)
capture_settings: Final = settings.get("accounting_capture")
if settings.get("accounting_sync_ack") and capture_settings is None:
raise ValueError("accounting_sync_ack requires accounting_capture")
if capture_settings is None:
if usage_capture.meter is not None:
raise ValueError("Disabling accounting_capture requires a worker restart")
return
if (
prisma_client is None
or CollectorSettings().enabled
or os.getenv("SPEND_LOGS_URL")
or litellm.apply_search_free_tier
):
raise ValueError(
"Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier"
)
from litellm.proxy.db.accounting_ledger import AccountingLedger
ledger: Final = AccountingLedger.for_client(prisma_client)
writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch)
usage_capture.start(
writer.write, UsageCaptureSettings.model_validate(capture_settings), load=ledger.load_usage_batch
)
def cost_tracking():
global prisma_client, spend_event_producer
_configure_usage_capture()
if prisma_client is not None:
from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
@ -7375,20 +7409,24 @@ class ProxyConfig:
verbose_proxy_logger.warning(
"Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k
)
router = litellm.Router(
**router_params,
assistants_config=assistants_config,
search_tools=search_tools,
router_general_settings=RouterGeneralSettings(
async_only_mode=True # only init async clients
),
ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid
fallback_access_check=router_fallback_access_check,
fallback_budget_check=RouterFallbackBudgetCheck(
is_enforced=router_fallback_budget_check.is_enforced,
team_member_model_budget_limiter=model_max_budget_limiter,
),
auto_router_capability_limit=_license_check.auto_router_capability_limit,
router = litellm.Router.from_config(
build_router_config(
router_arguments(
**router_params,
assistants_config=assistants_config,
search_tools=search_tools,
router_general_settings=RouterGeneralSettings(
async_only_mode=True # only init async clients
),
ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid
fallback_access_check=router_fallback_access_check,
fallback_budget_check=RouterFallbackBudgetCheck(
is_enforced=router_fallback_budget_check.is_enforced,
team_member_model_budget_limiter=model_max_budget_limiter,
),
auto_router_capability_limit=_license_check.auto_router_capability_limit,
)
)
)
if redis_usage_cache is not None and router.health().store.redis_cache is None:
@ -7878,19 +7916,23 @@ class ProxyConfig:
# Router can function with model_list=[] if search_tools are configured
if len(_model_list) > 0 or search_tools:
verbose_proxy_logger.debug("_model_list: %s", _model_list)
llm_router = litellm.Router(
model_list=_model_list,
router_general_settings=RouterGeneralSettings(
async_only_mode=True # only init async clients
),
search_tools=search_tools,
ignore_invalid_deployments=True,
fallback_access_check=router_fallback_access_check,
fallback_budget_check=RouterFallbackBudgetCheck(
is_enforced=router_fallback_budget_check.is_enforced,
team_member_model_budget_limiter=model_max_budget_limiter,
),
auto_router_capability_limit=_license_check.auto_router_capability_limit,
llm_router = litellm.Router.from_config(
build_router_config(
router_arguments(
model_list=_model_list,
router_general_settings=RouterGeneralSettings(
async_only_mode=True # only init async clients
),
search_tools=search_tools,
ignore_invalid_deployments=True,
fallback_access_check=router_fallback_access_check,
fallback_budget_check=RouterFallbackBudgetCheck(
is_enforced=router_fallback_budget_check.is_enforced,
team_member_model_budget_limiter=model_max_budget_limiter,
),
auto_router_capability_limit=_license_check.auto_router_capability_limit,
)
)
)
verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
else:
@ -13006,8 +13048,10 @@ async def completion(
"""
global user_temperature, user_request_timeout, user_max_tokens, user_api_base
data = {}
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
try:
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
base_llm_response_processor.data = data
if user_api_key_dict is not None:
if data.get("metadata") is None:
data["metadata"] = {}
@ -13021,7 +13065,6 @@ async def completion(
data["metadata"]["user_api_key_org_alias"] = user_api_key_dict.organization_alias
if hasattr(user_api_key_dict, "agent_id") and user_api_key_dict.agent_id is not None:
data["metadata"]["agent_id"] = user_api_key_dict.agent_id
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
return await base_llm_response_processor.base_process_llm_request(
request=request,
fastapi_response=fastapi_response,
@ -13123,9 +13166,12 @@ async def completion(
return _response
except Exception as e:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
user_api_key_dict=user_api_key_dict,
original_exception=e,
request_data=base_llm_response_processor.data,
dispatched_call_type=base_llm_response_processor.route_type,
)
litellm_call_id: Final = request_litellm_call_id(data)
litellm_call_id: Final = base_llm_response_processor.litellm_call_id
log_llm_api_exception(e, litellm_call_id)
if isinstance(e, ProxyException):
raise with_litellm_call_id(e, litellm_call_id)

View file

@ -36,7 +36,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
from litellm.caching.dual_cache import DualCache
from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.exceptions import EndpointNotSupportedByModelError, RateLimitError
from litellm.integrations.custom_guardrail import ModifyResponseException, guardrail_request_data_with_streaming
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
@ -103,6 +103,7 @@ from litellm.responses.streaming_iterator import (
ResponsesWsFrameGuard,
ResponsesWsFrameRejection,
spawn_logging_task,
turn_refusal_frame,
)
from litellm.router import Router
from litellm.types.llms.base import LiteLLMBaseModel
@ -2627,6 +2628,12 @@ async def _responses_websocket_session(
route_type="_aresponses_websocket",
skip_guardrails=True,
)
except RateLimitError as e:
verbose_proxy_logger.info("Responses WebSocket refused at open by a rate limit: %s", e)
with contextlib.suppress(WebSocketDisconnect, RuntimeError, OSError):
await websocket.send_text(turn_refusal_frame(e))
await websocket.close(code=1008, reason="Rate limit exceeded")
return
except Exception as e:
verbose_proxy_logger.exception("Responses WebSocket pre-call error")
try:

View file

@ -18,7 +18,9 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.router import router_arguments
from litellm.router_api import router_init_args
from litellm.router_config import build_router_config
# Client-supplied params that make the router or the call path fabricate a
# failure or a delay instead of calling the provider. The ``mock_testing_*``
@ -98,7 +100,9 @@ async def _route_user_config_request(data: dict, route_type: str):
filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config))
cached_router: Final = user_config_routers.get(
filtered_config, lambda: litellm.Router.from_config(build_router_config(router_arguments(**filtered_config)))
)
cached_router.observe(logging_obj)
return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj})

View file

@ -89,6 +89,7 @@ async def _finish_finalization(
if litellm.apply_search_free_tier
else None
)
logging_obj.cost_allowance_allocation = allocation # rebind-ok: retain the allocation for durable capture
logging_obj.cost_allowance_discount = ( # rebind-ok: retain allocation across snapshot retries
allocation.discount if allocation is not None else 0.0
)

View file

@ -3,7 +3,7 @@ import json
import os
import re
import secrets
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime, timezone
from datetime import datetime as dt
from functools import reduce
@ -55,6 +55,8 @@ from litellm.litellm_core_utils.get_llm_provider_logic import declared_authentic
from litellm.litellm_core_utils.hidden_params import served_from_cache
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
from litellm.litellm_core_utils.litellm_logging import (
StandardLoggingPayloadSetup,
UsageObservation,
coerce_model_access_groups,
is_valid_sha256_hash,
request_model_access_groups_from_litellm_params,
@ -599,6 +601,7 @@ def get_logging_payload(
start_time: datetime,
end_time: datetime,
llm_router: "Router | None" = None,
capture_usage: Callable[[SpendLogsPayload, str | None], None] | None = None,
) -> SpendLogsPayload:
if kwargs is None:
kwargs = {}
@ -622,7 +625,9 @@ def get_logging_payload(
routing_metadata,
routing_litellm_params,
)
completion_start_time: Final = kwargs.get("completion_start_time", end_time)
completion_start_time: Final = TypeAdapter[datetime | None](datetime | None).validate_python(
kwargs.get("completion_start_time", end_time)
)
call_type: Final = kwargs.get("call_type")
cache_hit: Final = kwargs.get("cache_hit", False)
@ -644,7 +649,7 @@ def get_logging_payload(
_usage: Final = (
chat_usage_of_batch.model_dump()
if chat_usage_of_batch is not None
else response_obj_dict.get("usage", None) or {}
else StandardLoggingPayloadSetup.get_usage_as_dict(response_obj_dict)
)
if isinstance(_usage, litellm.Usage):
usage = dict(_usage)
@ -928,7 +933,9 @@ def get_logging_payload(
cache_hit=str(cache_hit),
startTime=_ensure_datetime_utc(start_time),
endTime=_ensure_datetime_utc(end_time),
completionStartTime=_ensure_datetime_utc(completion_start_time),
completionStartTime=(
_ensure_datetime_utc(completion_start_time) if completion_start_time is not None else None
),
model=model_name,
user=metadata.get("user_api_key_user_id", "") or "",
team_id=metadata.get("user_api_key_team_id", "") or "",
@ -984,6 +991,14 @@ def get_logging_payload(
payload.get("spend"),
)
if capture_usage is not None:
observed_usage: Final = (
StandardLoggingPayloadSetup.get_usage_as_dict({"usage": usage})
if usage and kwargs.get("_accounting_usage_observed") is not UsageObservation.UNKNOWN
else None
)
capture_usage(payload, safe_dumps(observed_usage) if observed_usage is not None else None)
# Explicitly clear large intermediate objects to reduce memory pressure
del response_obj_dict, usage, clean_metadata, additional_usage_values

File diff suppressed because it is too large Load diff

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio
from collections import deque
from collections.abc import Awaitable, Callable
from collections.abc import Awaitable, Callable, Mapping
from concurrent.futures import Future
from dataclasses import dataclass
from itertools import accumulate, chain, count, takewhile
@ -114,6 +114,9 @@ class DurableUsageMeter:
self._admission: Final = Lock()
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
self._wakeup: Final = asyncio.Event()
self._capacity_changed: Final = asyncio.Event()
self._reservations: Mapping[object, tuple[int, int]] = {}
self._reserved_bytes = 0
self._task: asyncio.Task[None] | None = None
self._closed = False
self.pending_bytes = 0
@ -143,6 +146,36 @@ class DurableUsageMeter:
wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
return wrapped
def reserve_producer(self, size: int, headroom: int) -> object:
token: Final = object()
self.resize_producer(token, size, headroom)
return token
def resize_producer(self, token: object, size: int, headroom: int) -> None:
with self._admission:
previous: Final = self._reservations.get(token, (0, 0))[0]
delta: Final = size - previous
others: Final = (value[1] for key, value in self._reservations.items() if key is not token)
required: Final = max((headroom, *others))
if (
self._closed
or self.pending_bytes + delta > self.settings.max_pending_bytes
or self._reserved_bytes + delta + required > self.settings.max_pending_bytes
):
raise UsageCaptureError("Batch capture producer byte capacity exhausted")
self._reservations = {**self._reservations, token: (size, headroom)}
self._reserved_bytes += delta
self.pending_bytes += delta
self._capacity_changed.set()
def release_producer(self, token: object) -> None:
with self._admission:
size, _ = self._reservations[token]
self._reservations = {key: value for key, value in self._reservations.items() if key is not token}
self._reserved_bytes -= size
self.pending_bytes -= size
self._capacity_changed.set()
def _ensure_worker(self) -> None:
if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
self._wakeup.set()
@ -153,6 +186,34 @@ class DurableUsageMeter:
envelope: Final = '{"event":' + event.model_dump_json() + "}"
await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),)))
async def record_writes(self, writes: tuple[UsageWrite, ...]) -> None:
size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
capacity: Final = self.settings.max_pending_bytes - self._reserved_bytes
if len(writes) > min(self.settings.batch_size, self.settings.max_pending_records) or (
size > capacity and len(writes) > 1
):
middle: Final = len(writes) // 2
await self.record_writes(writes[:middle])
await self.record_writes(writes[middle:])
return
if size > capacity:
raise UsageCaptureError("One usage observation exceeds capture capacity")
while (receipt := self._submit_available(writes)) is None:
self._wakeup.set()
await self._capacity_changed.wait()
self._wakeup.set()
await asyncio.shield(receipt)
def _submit_available(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None] | None:
self._capacity_changed.clear()
try:
receipt: Final = self.submit(writes)
except UsageCaptureError:
if self._closed or not self.pending_records or not writes:
raise
return None
return receipt
async def flush(self) -> None:
self._ensure_worker()
self._wakeup.set()
@ -240,6 +301,7 @@ class DurableUsageMeter:
with self._admission:
self.pending_bytes -= item.size
self.pending_records -= len(item.writes)
self._capacity_changed.set()
if item.committed.done():
continue
if error is None:

View file

@ -163,7 +163,12 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.deployment_pricing import pricing_models
from litellm.litellm_core_utils.duration_parser import get_budget_window_start
from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
from litellm.litellm_core_utils.litellm_logging import (
BilledRoundsDispatch,
Logging,
UsageObservation,
deployment_pricing_model_info,
)
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -1739,6 +1744,39 @@ def _failure_fields_to_lift(
if _prompt_estimate is not None
else ()
)
from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation
_recorded_observation: Final[object] = _model_call_details.get("_accounting_usage_observed")
_current_usage: Final = (
_current_failure_usage_to_lift(
_model_call_details,
request_data,
_model_call_details.get("current_attempt_api_call_start_time", _first_handoff) is not None,
_billed,
_deployment_id,
_base_model,
dispatched_call_type,
)
if Logging.usage_capture_callback is not None
and (_billed or _model_call_details.get("combined_usage_object") is not None)
else None
)
_current_observation: Final = (
_recorded_observation
if isinstance(_recorded_observation, UsageObservation)
else UsageObservation.UNKNOWN
if _current_usage is None
else UsageObservation.KNOWN
if _model_call_details.get("combined_usage_object") is not None
else UsageObservation.ESTIMATED
)
_capture_observation: Final = UsageCaptureObservation.from_usage(
_current_usage[0] if _current_usage is not None and isinstance(_current_usage[0], Usage) else None,
_current_observation,
float(_current_usage[1])
if _current_usage is not None and isinstance(_current_usage[1], (int, float))
else None,
)
_entries: Final = (
("start_time", _model_call_details.get("start_time")),
("first_api_call_start_time", _first_handoff),
@ -1747,6 +1785,16 @@ def _failure_fields_to_lift(
("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)),
("standard_logging_object", _model_call_details.get("standard_logging_object")),
("billed_failed_attempts", _earlier_attempts or None),
("_accounting_capture", _model_call_details.get("_accounting_capture")),
("_accounting_pricing_snapshot", _model_call_details.get("_accounting_pricing_snapshot")),
("_accounting_finalized_cost", _logging_obj.finalized_cost if isinstance(_logging_obj, Logging) else None),
(
"_accounting_allocation",
_logging_obj.cost_allowance_allocation if isinstance(_logging_obj, Logging) else None,
),
("_accounting_usage_observed", _current_observation if Logging.usage_capture_callback is not None else None),
("_accounting_usage_snapshot", _capture_observation if Logging.usage_capture_callback is not None else None),
("_accounting_prior_captures", _model_call_details.get("_accounting_prior_captures")),
)
return MappingProxyType({key: value for key, value in _entries if value is not None})

View file

@ -223,6 +223,10 @@ async def _execute_query_pipeline(
"""
Execute the RAG query pipeline.
"""
parent_request_id: Final[str | None] = TypeAdapter[str | None](str | None).validate_python(
TypeAdapter(dict[str, object]).validate_python(kwargs).get("litellm_call_id")
)
# Extract router from kwargs - use it for completion if available
# to properly resolve virtual model names
router: Final[Router | None] = kwargs.pop("router", None)
@ -252,7 +256,7 @@ async def _execute_query_pipeline(
forwarded_search_params: Final = MappingProxyType(
{**provider_search_params, **kwargs, **filter_search_params, **store_search_params}
)
with internal_sub_call():
with internal_sub_call(parent_request_id):
search_response: Final = await litellm.vector_stores.asearch(
vector_store_id=retrieval_config["vector_store_id"],
query=query_text,
@ -286,7 +290,7 @@ async def _execute_query_pipeline(
if rerank and rerank.get("enabled"):
documents: Final = RAGQuery.extract_documents_from_search(search_response)
if documents:
with internal_sub_call():
with internal_sub_call(parent_request_id):
rerank_response = await litellm.arerank(
model=rerank["model"],
query=query_text,
@ -304,7 +308,7 @@ async def _execute_query_pipeline(
modified_messages: Final = messages[:-1] + [context_message] + [messages[-1]]
# Use router if available to properly resolve virtual model names
with internal_sub_call():
with internal_sub_call(parent_request_id):
if router is not None:
response = await router.acompletion(
model=model,

View file

@ -10,6 +10,7 @@ from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
@ -148,6 +149,9 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
),
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
@ -240,5 +244,8 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=responses_api_request,
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
litellm_metadata=kwargs.get("litellm_metadata", {}),
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
),
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")

View file

@ -200,6 +200,7 @@ class ResponsesSessionHandler:
previous_response_id: str,
item_cache_control: bool = False,
client_response_ids: tuple[str, ...] = (),
tool_call_provider_fields: bool = False,
stored_turns: StoredTurns | None = None,
) -> ChatCompletionSession:
"""
@ -241,9 +242,13 @@ class ResponsesSessionHandler:
for index in range(start.index, replayed_turns)
],
item_cache_control,
tool_call_provider_fields,
)
history_before_replay: Final = ResponsesSessionHandler._history_of_turns(
all_spend_logs[: start.index], proxy_server_requests[: start.index], item_cache_control
all_spend_logs[: start.index],
proxy_server_requests[: start.index],
item_cache_control,
tool_call_provider_fields,
)
chat_completion_message_history: Final = (
list(with_stored_calls_the_input_answers(replayed_history, history_before_replay))
@ -305,6 +310,7 @@ class ResponsesSessionHandler:
spend_logs: Sequence["SpendLogsPayload"],
proxy_server_requests: Sequence[Mapping[str, object] | None],
item_cache_control: bool,
tool_call_provider_fields: bool,
) -> list["SessionMessage"]:
return list(
chain.from_iterable(
@ -313,6 +319,7 @@ class ResponsesSessionHandler:
proxy_server_request_dict=proxy_server_request,
chat_completion_message_history=[],
item_cache_control=item_cache_control,
tool_call_provider_fields=tool_call_provider_fields,
)
for spend_log, proxy_server_request in zip(spend_logs, proxy_server_requests, strict=True)
)
@ -330,6 +337,7 @@ class ResponsesSessionHandler:
| Message
],
item_cache_control: bool = False,
tool_call_provider_fields: bool = False,
):
"""
Extend the chat completion message history with the spend log payload
@ -362,6 +370,7 @@ class ResponsesSessionHandler:
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,
tool_call_provider_fields=tool_call_provider_fields,
only_since_compaction=True,
)
)
@ -377,6 +386,7 @@ class ResponsesSessionHandler:
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,
tool_call_provider_fields=tool_call_provider_fields,
only_since_compaction=True,
)
)

View file

@ -9,6 +9,7 @@ import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.llms.anthropic.messages.native.utils import openai_chat_refusal_text
from litellm.main import stream_chunk_builder
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
from litellm.responses.litellm_completion_transformation.custom_tools import (
@ -35,6 +36,7 @@ from litellm.types.llms.openai import (
ContentPartAddedEvent,
ContentPartDoneEvent,
ContentPartDonePartOutputText,
ContentPartDonePartRefusal,
FunctionCallArgumentsDeltaEvent,
FunctionCallArgumentsDoneEvent,
OutputItemAddedEvent,
@ -45,6 +47,8 @@ from litellm.types.llms.openai import (
ReasoningSummaryPartDoneEvent,
ReasoningSummaryTextDeltaEvent,
ReasoningSummaryTextDoneEvent,
RefusalDeltaEvent,
RefusalDoneEvent,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseIncompleteEvent,
@ -88,6 +92,20 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str |
)
def _tool_call_objects(tool_calls: object) -> tuple[object, ...]:
return tuple(tool_calls) if isinstance(tool_calls, list) else ()
def _tool_call_provider_specific_fields(tool_call: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(tool_call)
fields: Final[object] = (
mapping.get("provider_specific_fields")
if mapping is not None
else getattr(tool_call, "provider_specific_fields", None)
)
return {"provider_specific_fields": fields} if fields else {}
def _is_signed_thinking_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data"))
@ -110,6 +128,7 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool:
delta.function_call,
getattr(delta, "images", None),
getattr(delta, "audio", None),
openai_chat_refusal_text(delta),
)
return any(fields) or _delta_has_signed_thinking_block(delta)
@ -119,6 +138,13 @@ def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool:
return has_reasoning and bool(delta.content or delta.tool_calls)
_WEB_SEARCH_PROGRESS_EVENTS: Final = (
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
)
class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"""
Async iterator for processing streaming responses from the Responses API.
@ -132,6 +158,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
responses_api_request: ResponsesAPIOptionalRequestParams,
custom_llm_provider: str | None = None,
litellm_metadata: dict | None = None,
tool_call_provider_fields: bool = False,
):
self.model: str = model
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper
@ -139,6 +166,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request
self.custom_llm_provider: str | None = custom_llm_provider
self.litellm_metadata: dict | None = litellm_metadata or {}
self.tool_call_provider_fields: Final = tool_call_provider_fields
# Store lightweight dict snapshots for stream_chunk_builder to reduce
# repeated Pydantic attribute access in end-of-stream assembly.
self.collected_chat_completion_chunks: list[dict[str, object]] = []
@ -152,6 +180,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self.sent_output_text_done_event: bool = False
self.sent_output_content_part_done_event: bool = False
self.sent_output_item_done_event: bool = False
self._sent_refusal_events: bool = False
self._annotation_index: int = 0
self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = ()
self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
@ -490,10 +519,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
except Exception:
tool_calls = None
if not tool_calls or not isinstance(tool_calls, list):
return
for tc in tool_calls:
for tc in _tool_call_objects(tool_calls):
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not call_id_raw:
continue
@ -568,7 +594,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=output_index,
sequence_number=self._sequence_number,
item=BaseLiteLLMOpenAIResponseObject(**item_kwargs),
item=BaseLiteLLMOpenAIResponseObject(
**item_kwargs,
**(_tool_call_provider_specific_fields(tc) if self.tool_call_provider_fields else {}),
),
)
self._pending_tool_events.append(item_done_event)
@ -592,11 +621,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
}
),
)
for event_type, event_class in (
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
):
progress_events: Final = (
_WEB_SEARCH_PROGRESS_EVENTS[:1] if item.status == "in_progress" else _WEB_SEARCH_PROGRESS_EVENTS
)
for event_type, event_class in progress_events:
self._sequence_number += 1
event = event_class(type=event_type, output_index=output_index, item_id=item.id)
event.__dict__["sequence_number"] = self._sequence_number
@ -798,9 +826,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
return event
def _queue_message_item_added_events(self) -> None:
if not self.sent_message_item_added_event:
self._queue_message_item_added_event()
self.sent_content_part_added_event = True
self._pending_response_events.append(self.create_content_part_added_event())
def _queue_message_item_added_event(self) -> None:
item_id: Final = self._message_item_id()
self.sent_message_item_added_event = True
self.sent_content_part_added_event = True
self._sequence_number += 1
event: Final = OutputItemAddedEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
@ -817,7 +850,47 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
)
event.__dict__["sequence_number"] = self._sequence_number
self._pending_response_events.append(event)
self._pending_response_events.append(self.create_content_part_added_event())
def _queue_refusal_events(self, refusal: str) -> None:
item_id: Final = self._message_item_id()
output_index: Final = self._message_index()
content_index: Final = 1 if self.sent_content_part_added_event else 0
events: Final = (
ContentPartAddedEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
item_id=item_id,
output_index=output_index,
content_index=content_index,
part=ContentPartDonePartRefusal(type="refusal", refusal=""),
),
RefusalDeltaEvent(
type=ResponsesAPIStreamEvents.REFUSAL_DELTA,
item_id=item_id,
output_index=output_index,
content_index=content_index,
delta=refusal,
),
RefusalDoneEvent(
type=ResponsesAPIStreamEvents.REFUSAL_DONE,
item_id=item_id,
output_index=output_index,
content_index=content_index,
refusal=refusal,
),
ContentPartDoneEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
item_id=item_id,
output_index=output_index,
content_index=content_index,
part=ContentPartDonePartRefusal(type="refusal", refusal=refusal),
),
)
first_sequence_number: Final = self._sequence_number + 1
self._sequence_number += len(events)
self._pending_response_events.extend(
event.model_copy(update={"sequence_number": sequence_number})
for sequence_number, event in enumerate(events, start=first_sequence_number)
)
def _merge_provider_specific_fields(self, src: dict) -> None:
"""Merge provider_specific_fields using last-value-wins for lists.
@ -974,14 +1047,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if self._cached_item_id is None:
self._cached_item_id = f"msg_{uuid.uuid4()}"
text: Final = litellm_complete_object.choices[0].message.content or ""
annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None)
message: Final = litellm_complete_object.choices[0].message
text: Final = message.content or ""
annotations = getattr(message, "annotations", None)
response_annotations: Final = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
annotations=annotations
)
)
text_parts: Final = (
({"type": "output_text", "text": text, "annotations": response_annotations},)
if self.sent_content_part_added_event
else ()
)
refusal: Final = openai_chat_refusal_text(message)
refusal_parts: Final = ({"type": "refusal", "refusal": refusal},) if refusal is not None else ()
return OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=self._message_index(),
@ -992,13 +1073,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"status": "completed",
"type": "message",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": text,
"annotations": response_annotations,
}
],
"content": [*text_parts, *refusal_parts],
}
),
)
@ -1050,22 +1125,33 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def return_default_done_events(
self, litellm_complete_object: ModelResponse
) -> BaseLiteLLMOpenAIResponseObject | None:
message: Final = litellm_complete_object.choices[0].message
refusal: Final = openai_chat_refusal_text(message)
if self.sent_message_item_added_event is False:
final_content: Final = litellm_complete_object.choices[0].message.content or ""
if not final_content:
final_content: Final = message.content or ""
if not final_content and refusal is None:
self.sent_output_text_done_event = True
self.sent_output_content_part_done_event = True
self.sent_output_item_done_event = True
return None
self._queue_message_item_added_events()
if final_content:
self._queue_message_item_added_events()
else:
self._queue_message_item_added_event()
return self._pending_response_events.pop(0)
if self.sent_output_text_done_event is False:
if self.sent_output_text_done_event is False and self.sent_content_part_added_event:
self.sent_output_text_done_event = True
return self.create_output_text_done_event(litellm_complete_object)
if self.sent_output_content_part_done_event is False:
if self.sent_output_content_part_done_event is False and self.sent_content_part_added_event:
self.sent_output_content_part_done_event = True
return self.create_output_content_part_done_event(litellm_complete_object)
if refusal is not None and not self._sent_refusal_events:
self._sent_refusal_events = True
self._queue_refusal_events(refusal)
return self._pending_response_events.pop(0)
if self.sent_output_item_done_event is False:
self.sent_output_text_done_event = True
self.sent_output_content_part_done_event = True
self.sent_output_item_done_event = True
return self.create_output_item_done_event(litellm_complete_object)
return None
@ -1169,7 +1255,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# DO NOT create message item
return
# Default: message
if openai_chat_refusal_text(delta) is not None and not delta.content:
self._queue_message_item_added_event()
return
self._queue_message_item_added_events()
return
@ -1413,7 +1501,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# Priority 2: Handle text deltas
delta_content: Final = self._get_delta_string_from_streaming_choices(chunk.choices)
if delta_content:
if not self.sent_message_item_added_event:
if not self.sent_content_part_added_event:
self._queue_message_item_added_events()
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
self._sequence_number += 1

View file

@ -8,7 +8,7 @@ import uuid
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from itertools import chain, groupby
from itertools import accumulate, chain, groupby
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -28,7 +28,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
from openai.types.chat.chat_completion_named_tool_choice_param import (
Function as NamedToolChoiceFunction,
)
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch, ResponseOutputRefusal
from openai.types.responses.response_compaction_item import ResponseCompactionItem
from openai.types.responses.response_create_params import ResponseInputParam
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
@ -54,7 +54,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
whole_messages_within_cache_control_limit,
with_cache_control,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
@ -64,7 +64,9 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
@ -373,8 +375,9 @@ class _ResponsesToolCallItem(Protocol):
def get(self, key: str, /) -> object: ...
class _MarkedToolCallChunk(ChatCompletionToolCallChunk):
class _ReplayedToolCallChunk(ChatCompletionToolCallChunk, total=False):
cache_control: ReadOnly[ChatCompletionCachedContent]
provider_specific_fields: ReadOnly[Mapping[str, object]]
class _MarkedToolMessage(ChatCompletionToolMessage):
@ -446,6 +449,32 @@ def _input_item_role(input_item: Mapping[str, object]) -> str:
return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated
def _starts_a_turn(input_item: Mapping[str, object]) -> bool:
item_type: Final = input_item.get("type")
if item_type in TOOL_CALL_OUTPUT_ITEM_TYPES:
return True
return item_type in (None, "message") and input_item.get("role") != "assistant"
def _is_unrun_hosted_search(input_item: Mapping[str, object]) -> bool:
return input_item.get("type") == "web_search_call" and input_item.get("status") == "in_progress"
def _without_encrypted_content(input_item: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in input_item.items() if key != "encrypted_content"}
def _without_paused_turn_thinking(input_items: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
turns: Final = tuple(accumulate(int(_starts_a_turn(item)) for item in input_items))
paused_turns: Final = frozenset(
turn for turn, item in zip(turns, input_items, strict=True) if _is_unrun_hosted_search(item)
)
return tuple(
_without_encrypted_content(item) if turn in paused_turns and item.get("type") == "reasoning" else item
for turn, item in zip(turns, input_items, strict=True)
)
SessionMessage: TypeAlias = (
AllMessageValues
| GenericChatCompletionMessage
@ -611,17 +640,28 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
return None
def _without_cache_control_key(item: object) -> object:
def _without_keys(item: object, keys: frozenset[str]) -> object:
if not isinstance(item, dict):
return item
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"}
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key not in keys}
def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam:
if not isinstance(input, list):
def _without_item_keys(input: str | ResponseInputParam, keys: frozenset[str]) -> str | ResponseInputParam:
if not keys or not isinstance(input, list):
return input
return cast( # cast-ok: the same input items, each minus its own cache_control key
ResponseInputParam, [_without_cache_control_key(item) for item in input]
return cast( # cast-ok: the same input items, each minus the given keys
ResponseInputParam, [_without_keys(item, keys) for item in input]
)
def _dropped_item_keys(item_cache_control: bool, tool_call_provider_fields: bool) -> frozenset[str]:
return frozenset(
key
for key, kept in (
("cache_control", item_cache_control),
("provider_specific_fields", tool_call_provider_fields),
)
if not kept
)
@ -1114,6 +1154,7 @@ class LiteLLMCompletionResponsesConfig:
responses_api_request=responses_api_request,
replay_reasoning=True,
item_cache_control=keeps_item_marks,
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
native_compaction=native_compaction,
only_since_compaction=not native_compaction,
)
@ -1177,6 +1218,7 @@ class LiteLLMCompletionResponsesConfig:
responses_api_request: ResponsesAPIOptionalRequestParams | dict,
replay_reasoning: bool = False,
item_cache_control: bool = False,
tool_call_provider_fields: bool = False,
native_compaction: bool = False,
only_since_compaction: bool = False,
) -> list[
@ -1194,7 +1236,9 @@ class LiteLLMCompletionResponsesConfig:
assistant ``reasoning_content`` and signed ``thinking_blocks`` so the
provider gets its own chain-of-thought back instead of reading it as
visible text. ``item_cache_control`` belongs to the same callers, when
the target keeps an input item's ``cache_control`` on the message.
the target keeps an input item's ``cache_control`` on the message, and
``tool_call_provider_fields`` when the target reads a function call's
thought signature from ``provider_specific_fields`` instead of its id.
Callers that only inspect the messages (token counting, rate limiting,
guardrail scanning) leave both off, because they need every piece of text
@ -1232,7 +1276,7 @@ class LiteLLMCompletionResponsesConfig:
read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction)
messages.extend(
LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
input=read_input if item_cache_control else without_item_cache_control(read_input),
input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)),
replay_reasoning=replay_reasoning,
native_compaction=native_compaction,
)
@ -1290,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig:
ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id,
item_cache_control=keeps_item_marks,
client_response_ids=previous_response_client_ids,
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
stored_turns=stored_turns,
)
chat_completion_session: Final = (
@ -1416,7 +1461,7 @@ class LiteLLMCompletionResponsesConfig:
messages.append(ChatCompletionUserMessage(role="user", content=input))
elif isinstance(input, list):
existing_tool_call_ids: Final[set[str]] = set()
for _input in input:
for _input in _without_paused_turn_thinking(input):
chat_completion_messages = (
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
input_item=_input,
@ -2473,7 +2518,7 @@ class LiteLLMCompletionResponsesConfig:
tool_call: Final = (
ChatCompletionToolCallChunk(id=tool_call_id, type="function", function=tool_call_function, index=0)
if function_call_cache_control is None
else _MarkedToolCallChunk(
else _ReplayedToolCallChunk(
id=tool_call_id,
type="function",
function=tool_call_function,
@ -2481,9 +2526,17 @@ class LiteLLMCompletionResponsesConfig:
cache_control=function_call_cache_control,
)
)
provider_specific_fields: Final = as_str_mapping(function_call.get("provider_specific_fields"))
replayed_tool_call: Final = (
tool_call
if provider_specific_fields is None
else cast( # cast-ok: the same tool call plus the provider_specific_fields the client sent back
_ReplayedToolCallChunk, {**tool_call, "provider_specific_fields": provider_specific_fields}
)
)
# Create an assistant message with the tool call
return [ChatCompletionResponseMessage(tool_calls=[tool_call], role="assistant", content=None)]
return [ChatCompletionResponseMessage(tool_calls=[replayed_tool_call], role="assistant", content=None)]
@staticmethod
def _transform_input_file_item_to_file_item(item: Mapping[str, object]) -> _ResponsesFilePart:
@ -2528,45 +2581,55 @@ class LiteLLMCompletionResponsesConfig:
for item in content:
if isinstance(item, str):
content_list.append(item)
elif isinstance(item, dict):
if (
media := decode_responses_media(
fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
elif (
isinstance(item, dict)
and (
part
:= LiteLLMCompletionResponsesConfig._transform_responses_api_content_part_to_chat_completion_part(
_validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
)
) is not None:
content_list.append(dict(media))
elif fields.get("type") == "input_image":
content_list.append(dict(_responses_image_part(fields)))
elif fields.get("type") == "input_file":
content_list.append(dict(_responses_file_part(fields)))
elif fields.get("type") == "input_audio":
content_list.append(dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields)))
elif fields.get("type") == "encrypted_content":
encrypted_content = fields.get("encrypted_content")
if encrypted_content is not None:
content_list.append(
OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content))
)
else:
# Skip text blocks with None text to avoid downstream errors
text_value = fields.get("text")
if text_value is None:
continue
content_block: dict[str, object] = {
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
content_type
if isinstance(content_type := fields.get("type"), str) and content_type
else "text"
),
"text": text_value,
}
if "cache_control" in fields:
content_block["cache_control"] = fields["cache_control"]
content_list.append(content_block)
)
is not None
):
content_list.append(part)
return content_list
else:
raise ValueError(f"Invalid content type: {type(content)}")
@staticmethod
def _transform_responses_api_content_part_to_chat_completion_part(
fields: dict[str, object],
) -> dict[str, object] | None:
if (media := decode_responses_media(fields)) is not None:
return dict(media)
part_type: Final = fields.get("type")
if part_type == "input_image":
return dict(_responses_image_part(fields))
if part_type == "input_file":
return dict(_responses_file_part(fields))
if part_type == "input_audio":
return dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields))
if part_type == "refusal":
refusal: Final = fields.get("refusal")
return dict(OpenAIChatCompletionTextObject(type="text", text=refusal)) if isinstance(refusal, str) else None
if part_type == "encrypted_content":
encrypted_content: Final = fields.get("encrypted_content")
return (
None
if encrypted_content is None
else dict(OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content)))
)
text_value: Final = fields.get("text")
if text_value is None:
return None
return {
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
part_type if isinstance(part_type, str) and part_type else "text"
),
"text": text_value,
**({"cache_control": fields["cache_control"]} if "cache_control" in fields else {}),
}
@staticmethod
def _get_chat_completion_request_content_type(
content_type: str,
@ -3045,6 +3108,18 @@ class LiteLLMCompletionResponsesConfig:
# Default to completed for unknown finish reasons
return "completed"
@staticmethod
def _is_paused_turn(choice: Choices) -> bool:
return anthropic_stop_reason(choice.message) == "pause_turn"
@staticmethod
def _responses_status_for_choice(choice: Choices) -> ResponsesAPIStatus:
if LiteLLMCompletionResponsesConfig._is_paused_turn(choice):
return "completed"
return LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
)
@staticmethod
def _incomplete_details_for_finish_reason(
finish_reason: str | None,
@ -3181,19 +3256,21 @@ class LiteLLMCompletionResponsesConfig:
"""
if isinstance(chat_completion_response, dict):
chat_completion_response = ModelResponse(**chat_completion_response)
# Get finish_reason from the first choice to determine overall status
finish_reason: str | None = None
choices: Final[list[Choices]] = getattr(chat_completion_response, "choices", [])
if choices and len(choices) > 0:
finish_reason = choices[0].finish_reason
finish_reason: Final = choices[0].finish_reason if choices else None
paused_turn: Final = bool(choices) and LiteLLMCompletionResponsesConfig._is_paused_turn(choices[0])
echoed: Final = _echoable_request_params(responses_api_request)
chat_usage: Final = getattr(chat_completion_response, "usage", None)
incomplete_details: Final = LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
finish_reason=finish_reason,
existing=getattr(chat_completion_response, "incomplete_details", None),
completion_tokens=getattr(chat_usage, "completion_tokens", None),
max_output_tokens=echoed.get("max_output_tokens"),
incomplete_details: Final = (
None
if paused_turn
else LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
finish_reason=finish_reason,
existing=getattr(chat_completion_response, "incomplete_details", None),
completion_tokens=getattr(chat_usage, "completion_tokens", None),
max_output_tokens=echoed.get("max_output_tokens"),
)
)
responses_api_response: Final[ResponsesAPIResponse] = ResponsesAPIResponse(
@ -3220,8 +3297,8 @@ class LiteLLMCompletionResponsesConfig:
max_output_tokens=echoed.get("max_output_tokens"),
previous_response_id=echoed.get("previous_response_id"),
reasoning=echoed.get("reasoning"),
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
finish_reason
status=(
LiteLLMCompletionResponsesConfig._responses_status_for_choice(choices[0]) if choices else "incomplete"
),
text=echoed.get("text") or {},
truncation=echoed.get("truncation"),
@ -3367,9 +3444,7 @@ class LiteLLMCompletionResponsesConfig:
GenericResponseOutputItem(
type="reasoning",
id=mint_reasoning_item_id(),
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
),
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
role="assistant",
content=[
OutputText(
@ -3492,24 +3567,34 @@ class LiteLLMCompletionResponsesConfig:
choice=choice,
)
message_output_items.extend(image_generation_items)
elif choice.message.content is not None or getattr(choice.message, "annotations", None):
elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message):
message_output_items.append(
GenericResponseOutputItem(
type="message",
id=f"msg_{uuid.uuid4()}",
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
),
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
role=choice.message.role,
content=[
LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(
choice.message
)
],
content=list(content),
)
)
return message_output_items
@staticmethod
def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]:
refusal: Final = openai_chat_refusal_text(message)
refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else ()
has_text_part: Final = (
bool(message.content)
or bool(getattr(message, "annotations", None))
or (message.content is not None and not refusal_parts)
)
text_parts: Final = (
(LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),)
if has_text_part
else ()
)
return (*text_parts, *refusal_parts)
@staticmethod
def _transform_responses_api_outputs_to_chat_completion_messages(
responses_api_output: ResponsesAPIResponse,

View file

@ -437,7 +437,7 @@ async def aresponses_api_with_mcp(
#########################################################
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.carry_failed_attempt_usage()
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
response: Final = await aresponses(
input=input,
model=model,
@ -514,7 +514,9 @@ async def aresponses_api_with_mcp(
tool_calls=tool_calls, tool_results=tool_results
)
with internal_sub_call():
with internal_sub_call(
logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
):
final_response = await LiteLLM_Proxy_MCP_Handler.make_follow_up_call(
follow_up_input=follow_up_input,
model=model,

View file

@ -663,7 +663,7 @@ async def acompletion_with_mcp(
logging_obj: Final = kwargs.get("litellm_logging_obj")
if isinstance(logging_obj, LiteLLMLoggingObj):
logging_obj.carry_failed_attempt_usage()
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
initial_response: Final = await litellm_acompletion(**initial_call_args)
first_round_breakdown: Final = logging_obj.cost_breakdown if isinstance(logging_obj, LiteLLMLoggingObj) else None
@ -713,7 +713,7 @@ async def acompletion_with_mcp(
follow_up_call_args["messages"] = follow_up_messages
follow_up_call_args["stream"] = stream
with internal_sub_call():
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
final_response: Final = await litellm_acompletion(**follow_up_call_args)
if not isinstance(final_response, ModelResponse):

View file

@ -35,11 +35,13 @@ import litellm
from litellm.constants import (
EMPTY_MAPPING,
LITELLM_MAX_STREAMING_DURATION_SECONDS,
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
STREAM_SSE_DONE_STRING,
)
from litellm.exceptions import MidStreamFallbackError, RateLimitError
from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
process_response_headers,
provider_reported_usage_cost,
set_response_cost_in_hidden_params,
@ -100,6 +102,12 @@ class ProjectQuotaCallback(Protocol):
) -> None: ...
class TokenLimitCallback(Protocol):
async def enforce_token_limits_for_frame(
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
) -> None: ...
@lru_cache(maxsize=1)
def get_openai_response_types():
from litellm.types.llms import openai as openai_types
@ -177,6 +185,7 @@ def _typed_gets_litellm_params(fn: _GetsLitellmParams) -> _GetsLitellmParams:
_SHOULD_STORE_RESULT_IN_CACHE_ATTR: Final = "_should_store_result_in_cache"
_UNMASK_PII_TEXT_ATTR: Final = "_unmask_pii_text"
_DECODED_JSON: Final = TypeAdapter(object)
_HIDDEN_PARAMS: Final = TypeAdapter(dict[str, object])
def _load_json_object(payload: str | bytes) -> dict[str, object]:
@ -224,6 +233,11 @@ def spawn_logging_task(
async def drain_logging_tasks(timeout: float = 5.0) -> None:
# A task that finished as its loop closed never ran its discard callback, and a
# task from another loop cannot be awaited here: forget both, wait on the rest.
running_loop: Final = asyncio.get_running_loop()
stale: Final = {task for task in _PENDING_LOGGING_TASKS if task.done() or task.get_loop() is not running_loop}
_PENDING_LOGGING_TASKS.difference_update(stale)
if not _PENDING_LOGGING_TASKS:
return
_, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout)
@ -686,7 +700,9 @@ class BaseResponsesAPIStreamingIterator:
source_hidden: Final[object] = getattr(
getattr(self.completed_response, "response", None), "_hidden_params", None
)
source: Final[Mapping[str, object]] = source_hidden if isinstance(source_hidden, Mapping) else EMPTY_MAPPING
source: Final[Mapping[str, object]] = (
_HIDDEN_PARAMS.validate_python(source_hidden) if isinstance(source_hidden, Mapping) else {}
)
processed: Final[object] = source.get("additional_headers") or self._hidden_params.get("additional_headers")
raw: Final[object] = source.get("headers") or self._raw_response_headers
headers: Final[Mapping[str, object]] = processed if isinstance(processed, Mapping) else EMPTY_MAPPING
@ -696,6 +712,7 @@ class BaseResponsesAPIStreamingIterator:
target.hidden_params = {
"additional_headers": {**headers},
"headers": {**raw_headers},
**_stamped_bill_hidden_params(source),
**existing,
}
@ -1742,6 +1759,11 @@ def _dump_json_safely(response: BaseModel) -> str | None:
return None
def _stamped_bill_hidden_params(source: Mapping[str, object]) -> Mapping[str, object]:
stamped: Final = UPSTREAM_USAGE_COST in source
return {key: source[key] for key in (UPSTREAM_USAGE_COST, "response_cost") if stamped and key in source}
def _logging_copy(event: object) -> object:
"""Hand logging callbacks a copy, so their usage rewrite (Responses shape to chat shape) never
reaches the event the caller is iterating. The round trip through ``model_dump`` sidesteps the
@ -2158,6 +2180,25 @@ class _RefusedTurn:
ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]]
def with_token_limit_checks(
turn_admission: ResponsesWsTurnAdmission | None,
token_limit_callbacks: Sequence[TokenLimitCallback],
user_api_key_dict: UserAPIKeyAuth | None,
requested_model: str | None,
) -> ResponsesWsTurnAdmission | None:
if not token_limit_callbacks or user_api_key_dict is None:
return turn_admission
async def admit() -> Mapping[str, object]:
for callback in token_limit_callbacks:
await callback.enforce_token_limits_for_frame(
user_api_key_dict=user_api_key_dict, requested_model=requested_model
)
return MappingProxyType({}) if turn_admission is None else await turn_admission()
return admit
async def _admit_frame_turn(
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str
) -> _AdmittedTurn | _RefusedTurn:
@ -2173,22 +2214,24 @@ async def _admit_frame_turn(
return _RefusedTurn(e)
def _turn_refusal_frame(refusal: RateLimitError) -> str:
def turn_refusal_frame(refusal: RateLimitError) -> str:
return json.dumps(
{"type": "error", "status": 429, "error": {"type": "rate_limit_exceeded", "message": str(refusal)}}
)
def _turn_admission_error_frame(failure: Exception) -> str:
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
def _turn_admission_error_frame(failure: Exception, status: int) -> str:
return json.dumps({"type": "error", "status": status, "error": {"type": "server_error", "message": str(failure)}})
def _turn_rejection_frame(rejection: Exception) -> str:
match rejection:
case RateLimitError():
return _turn_refusal_frame(rejection)
return turn_refusal_frame(rejection)
case ServiceUnavailableError():
return _turn_admission_error_frame(rejection, rejection.status_code)
case _:
return _turn_admission_error_frame(rejection)
return _turn_admission_error_frame(rejection, 500)
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
@ -3015,7 +3058,7 @@ class ResponsesWebSocketStreaming:
async def _send_turn_refusal(self, refusal: RateLimitError) -> None:
try:
await self.websocket.send_text(_turn_refusal_frame(refusal))
await self.websocket.send_text(turn_refusal_frame(refusal))
except Exception: # noqa: BLE001, S110 # client may already be gone
pass
@ -3569,7 +3612,7 @@ class ManagedResponsesWebSocketHandler:
self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
)
except RateLimitError as e:
await self._send_frame(_turn_refusal_frame(e))
await self._send_frame(turn_refusal_frame(e))
return
guarded: Final = await self._guard_frame(raw_message)
if guarded is None:
@ -3613,6 +3656,7 @@ class ManagedResponsesWebSocketHandler:
call_kwargs["litellm_metadata"] = {
**(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}),
**verdict.reservation,
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: self.logging_obj.litellm_call_id,
}
try:

File diff suppressed because it is too large Load diff

View file

@ -42,6 +42,8 @@ from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload
from litellm.router_config import detached
if TYPE_CHECKING:
from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type
@ -176,9 +178,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t
Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id``
set) only by that team, any other deployment by everyone. Needs no router instance.
"""
from litellm.router import Router
from litellm.router_python_core import PythonCore
return Router._core_deployment_usable_by_team(deployment, team_id)
return PythonCore._core_deployment_usable_by_team(deployment, team_id)
def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str:
@ -200,9 +202,9 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str
def router_init_args() -> tuple[str, ...]:
"""The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance."""
from litellm.router import Router
from litellm.router_python_core import PythonCore
return tuple(Router.get_valid_args())
return tuple(PythonCore.get_valid_args())
def mark_advisor_orchestration_failure(exception: BaseException) -> None:
@ -266,9 +268,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob
def _json_default_stable_id(obj: object) -> str:
from litellm.router import Router
from litellm.router_python_core import PythonCore
return Router._json_default_stable_id(obj)
return PythonCore._json_default_stable_id(obj)
class RouterRegistry:
@ -529,10 +531,10 @@ class RouterRegistry:
``should_include_deployment``), or, without ``model_name``, whether the team may use it at
all (was ``deployment_usable_by_team``).
"""
from litellm.router import Router
from litellm.router_python_core import PythonCore
if model_name is None:
return Router._core_deployment_usable_by_team(deployment, team_id)
return PythonCore._core_deployment_usable_by_team(deployment, team_id)
return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id)
@overload
@ -1053,6 +1055,12 @@ SETTING_FIELDS: Final[tuple[str, ...]] = (
"cache_responses",
)
# Runtime settings kept as plain router attributes rather than ``settings()`` fields. A write
# to one (``router.num_retries = 2``) is a changeset too: ``ChangeSet(assign={...})``.
PLAIN_SETTING_ATTRIBUTES: Final[frozenset[str]] = frozenset({"num_retries", "max_fallbacks", "retry_after", "timeout"})
# Former attributes whose value lives under another name, assignable through a changeset.
ASSIGNABLE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"model_group_alias": "_core_model_group_alias"})
_MISSING: Final = object()
@ -1168,16 +1176,16 @@ class RouterHealth:
@staticmethod
def is_blocked(deployment: "Deployment") -> bool:
"""Whether an admin paused this deployment (was ``is_deployment_blocked``)."""
from litellm.router import Router
from litellm.router_python_core import PythonCore
return Router._core_is_deployment_blocked(deployment)
return PythonCore._core_is_deployment_blocked(deployment)
@staticmethod
def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
"""Whether every deployment in the list is paused (was ``are_all_deployments_blocked``)."""
from litellm.router import Router
from litellm.router_python_core import PythonCore
return Router._core_are_all_deployments_blocked(deployments)
return PythonCore._core_are_all_deployments_blocked(deployments)
def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set
"""Groups whose every deployment is paused (was ``get_fully_blocked_model_names``)."""
@ -1296,13 +1304,17 @@ class RoutingObservers:
def apply_changes(router: "Router", changes: ChangeSet) -> ApplyResult:
if changes.refresh_model_info:
raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()")
unknown: Final = sorted(set(changes.assign) - set(SETTING_FIELDS))
unknown: Final = sorted(set(changes.assign) - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES})
if unknown:
raise ValueError(f"not assignable router settings: {unknown}")
for name, value in changes.assign.items():
setattr(router, "_core_" + name, value)
setattr(
router,
name if name in PLAIN_SETTING_ATTRIBUTES else ASSIGNABLE_ALIASES.get(name, "_core_" + name),
detached(value),
)
if changes.settings:
router._core_update_settings(**changes.settings)
router._core_update_settings(**{name: detached(value) for name, value in changes.settings.items()})
upserted: Final = [router._core_upsert_deployment(deployment=d) for d in changes.upsert]
deleted: Final = [router._core_delete_deployment(id=i) for i in changes.delete]
if changes.redis_cache is not None:
@ -1388,7 +1400,7 @@ class RoutingContext:
return self._router.observe()
def belongs_to(self, router: "Router") -> bool:
return self._router is router
return self._router is router or router.__dict__.get("_core") is self._router
@property
def explicit_allowed_fails(self) -> int | None:

157
litellm/router_config.py Normal file
View file

@ -0,0 +1,157 @@
"""
The router's configuration, built outside the router.
``RouterConfig`` is static: the constructor arguments, frozen for the router's life.
``RouterDynamicConfig`` is a snapshot of the process settings the router reads while it serves
(``litellm.num_retries``, ``litellm.model_cost``, the fallback lists, ...). A call reads one snapshot
at its start. This module is the only router code that reads ``litellm.*`` settings.
"""
import itertools
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import litellm
if TYPE_CHECKING:
from litellm.caching.caching import Cache
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
@dataclass(frozen=True, slots=True)
class RouterConfig:
"""The arguments the router was built with, exactly as passed (unset ones are absent)."""
arguments: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
def get(self, name: str, default: object = None) -> object:
return self.arguments.get(name, default)
@dataclass(frozen=True, slots=True)
class RouterDynamicConfig:
"""Process settings the router reads per call. Values are the live objects, not copies."""
version: int
num_retries: int | None
max_fallbacks: int | None
router_max_fallbacks: int
allowed_fails: int
request_timeout: float
fallbacks: Sequence[object] | None
default_fallbacks: Sequence[str] | None
context_window_fallbacks: Sequence[object] | None
content_policy_fallbacks: Sequence[object] | None
drop_params: bool | None
provider_list: Sequence[str]
expose_router_debug_in_errors: bool
redact_user_api_key_info: bool | None
model_alias_map: Mapping[str, str]
model_cost: Mapping[str, object]
known_custom_logger_compatible_callbacks: Sequence[str]
exception_types: Sequence[type[BaseException]]
callbacks: Sequence[object]
success_callback: Sequence[object]
failure_callback: Sequence[object]
input_callback: Sequence[object]
callback_manager: "LoggingCallbackManager"
cache: "Cache | None"
_versions: Final = itertools.count(1)
def detached(value: object) -> object:
"""``value`` with its plain containers (dict, list, tuple, set) copied, all the way down, so the
caller's own objects can't change the router afterwards. Other objects (clients, caches,
callbacks, pydantic models) are kept as they are."""
if isinstance(value, dict):
return {key: detached(item) for key, item in value.items()}
if isinstance(value, list):
return [detached(item) for item in value]
if isinstance(value, tuple) and type(value) is tuple:
return tuple(detached(item) for item in value)
if isinstance(value, set):
return {detached(item) for item in value}
return value
def build_router_config(arguments: Mapping[str, object]) -> RouterConfig:
"""The constructor arguments, detached from the caller's objects: settings are static."""
return RouterConfig(arguments=MappingProxyType({name: detached(value) for name, value in arguments.items()}))
def current_dynamic_config() -> RouterDynamicConfig:
"""A snapshot of the process settings as they are right now."""
return RouterDynamicConfig(
version=next(_versions),
num_retries=litellm.num_retries,
max_fallbacks=litellm.max_fallbacks,
router_max_fallbacks=litellm.ROUTER_MAX_FALLBACKS,
allowed_fails=litellm.allowed_fails,
request_timeout=litellm.request_timeout,
fallbacks=litellm.fallbacks,
default_fallbacks=litellm.default_fallbacks,
context_window_fallbacks=litellm.context_window_fallbacks,
content_policy_fallbacks=litellm.content_policy_fallbacks,
drop_params=litellm.drop_params,
provider_list=litellm.provider_list,
expose_router_debug_in_errors=litellm.expose_router_debug_in_errors,
redact_user_api_key_info=litellm.redact_user_api_key_info,
model_alias_map=litellm.model_alias_map,
model_cost=litellm.model_cost,
known_custom_logger_compatible_callbacks=litellm._known_custom_logger_compatible_callbacks,
exception_types=litellm.LITELLM_EXCEPTION_TYPES,
callbacks=litellm.callbacks,
success_callback=litellm.success_callback,
failure_callback=litellm.failure_callback,
input_callback=litellm.input_callback,
callback_manager=litellm.logging_callback_manager,
cache=litellm.cache,
)
_LIVE_SOURCES: Final = MappingProxyType(
{
"num_retries": "num_retries",
"max_fallbacks": "max_fallbacks",
"router_max_fallbacks": "ROUTER_MAX_FALLBACKS",
"allowed_fails": "allowed_fails",
"request_timeout": "request_timeout",
"fallbacks": "fallbacks",
"default_fallbacks": "default_fallbacks",
"context_window_fallbacks": "context_window_fallbacks",
"content_policy_fallbacks": "content_policy_fallbacks",
"drop_params": "drop_params",
"provider_list": "provider_list",
"expose_router_debug_in_errors": "expose_router_debug_in_errors",
"redact_user_api_key_info": "redact_user_api_key_info",
"model_alias_map": "model_alias_map",
"model_cost": "model_cost",
"known_custom_logger_compatible_callbacks": "_known_custom_logger_compatible_callbacks",
"exception_types": "LITELLM_EXCEPTION_TYPES",
"callbacks": "callbacks",
"success_callback": "success_callback",
"failure_callback": "failure_callback",
"input_callback": "input_callback",
"callback_manager": "logging_callback_manager",
"cache": "cache",
}
)
class LiveDynamicConfig:
"""``RouterDynamicConfig``'s fields read from the process settings at access time, with no snapshot."""
__slots__ = ()
def __getattr__(self, name: str) -> object:
source: Final = _LIVE_SOURCES.get(name)
if source is None:
raise AttributeError(name)
return getattr(litellm, source)
live_dynamic_config: Final = LiveDynamicConfig()

56
litellm/router_core.py Normal file
View file

@ -0,0 +1,56 @@
"""
``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one.
``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core
(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when
the router is built, from its ``RouterConfig``.
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Protocol, runtime_checkable
from litellm.router_config import RouterConfig
if TYPE_CHECKING:
from litellm.router_api import (
ApplyResult,
ChangeSet,
RouterHealth,
RouterRegistry,
RouterSettings,
RoutingObservers,
)
from litellm.router_python_core import PythonCore
from litellm.router_python_host import PythonHost
from litellm.types.router import CustomRoutingStrategyBase
@runtime_checkable
class RouterCore(Protocol):
async def call(self, op: str, request: Mapping[str, object]) -> object: ...
def registry(self) -> "RouterRegistry": ...
def settings(self) -> "RouterSettings": ...
def health(self) -> "RouterHealth": ...
async def select(
self,
model: str,
request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict
) -> object: ...
def apply(self, changes: "ChangeSet") -> "ApplyResult": ...
def observe(self) -> "RoutingObservers": ...
def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None: ...
def retire(self) -> None: ...
def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
from litellm.router_python_core import PythonCore
return (core_class or PythonCore)(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature

View file

@ -31,7 +31,7 @@ from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload
from litellm.router_api import SETTING_FIELDS, ChangeSet
from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet
if TYPE_CHECKING:
from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged
@ -65,7 +65,7 @@ _INTERNAL_PREFIX: Final = "litellm.router"
# Router's own attribute hooks: a write or delete through an old name passes through them first.
_ATTRIBUTE_HOOKS: Final = frozenset({"__setattr__", "__delattr__"})
_ATTRIBUTE_HOOKS: Final = frozenset({"__getattribute__", "__setattr__", "__delattr__"})
def _note(name: str, replacement: str) -> None:
@ -88,9 +88,10 @@ def _note(name: str, replacement: str) -> None:
def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
"""The Router this mixin is part of (it is mixed into ``litellm.Router`` only)."""
from litellm.router import Router
from litellm.router_python_core import PythonCore
if isinstance(obj, Router):
return obj
if isinstance(obj, (Router, PythonCore)):
return obj # pyright: ignore[reportReturnType] # a PythonCore carries every member the mixin reads
raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
@ -134,7 +135,7 @@ class _DeprecatedAttribute(Generic[_T]):
def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
_note(self.name, self.replacement)
if self.name in SETTING_FIELDS:
if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
_as_router(obj).apply(ChangeSet(assign={self.name: value}))
else:
setattr(obj, "_core_" + self.name, value)
@ -169,6 +170,13 @@ _OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] =
)
def _override_key(router: object, name: str) -> tuple[int, str]:
"""The running-override key for ``router``: a Router and its core share one, so an old member
entered on the Router is seen by the override routing that runs on the core."""
core: Final = getattr(router, "__dict__", {}).get("_core")
return (id(router if core is None else core), name)
def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]:
"""
A direct call of a subclass override (``router.<name>()``) marks it running, so its
@ -184,7 +192,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
*args: object,
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
) -> object:
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
try:
return await async_override(router, *args, **kwargs)
finally:
@ -198,7 +206,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
*args: object,
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
) -> object:
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
try:
return override(router, *args, **kwargs)
finally:
@ -236,7 +244,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
*args: object,
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
) -> object:
key: Final = (id(router), name)
key: Final = _override_key(router, name)
running: Final = _OVERRIDES_RUNNING.get()
if key in running:
parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core)
@ -255,7 +263,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
*args: object,
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
) -> object:
key: Final = (id(router), name)
key: Final = _override_key(router, name)
running: Final = _OVERRIDES_RUNNING.get()
if key in running:
return parent(router)(*args, **kwargs)
@ -502,9 +510,9 @@ class DeprecatedRouterMembers:
@staticmethod
def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool:
_note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)")
from litellm.router import Router
from litellm.router_python_core import PythonCore
return Router._core_deployment_usable_by_team(model, team_id)
return PythonCore._core_deployment_usable_by_team(model, team_id)
_deployment_usable_by_team = deployment_usable_by_team

17669
litellm/router_python_core.py Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,135 @@
"""
``PythonHost``: what the router core reaches outside itself, served from the litellm package.
The core never touches ``litellm.*`` directly. Inference calls (``invoke``), backoff sleeps,
callback registration and the process-wide cache all go through the host, and the process settings
come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the same operations.
"""
import asyncio
from collections.abc import Awaitable, Callable, Sequence
from typing import (
TYPE_CHECKING,
Final,
Literal,
TypeAlias,
cast, # noqa: TID251 # untyped litellm globals: the live config view and the callback lists
)
from typing_extensions import overload
import litellm
from litellm.router_config import RouterDynamicConfig, live_dynamic_config
if TYPE_CHECKING:
from pydantic import BaseModel
from litellm.caching.caching import Cache
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
AdapterCompletionStreamWrapper,
EmbeddingResponse,
ImageResponse,
LiteLLMBatch,
ModelInfo,
ModelResponse,
TextCompletionResponse,
TranscriptionResponse,
)
CallbackList: TypeAlias = Literal[
"success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback", "input_callback"
]
Endpoint: TypeAlias = Callable[..., object]
class PythonHost:
__slots__ = ()
# The endpoints whose results the router core reads, typed as litellm declares them; any other
# op gets the untyped Endpoint.
@overload
def invoke(self, op: Literal["acompletion"]) -> Callable[..., Awaitable["ModelResponse | CustomStreamWrapper"]]: ...
@overload
def invoke(
self, op: Literal["atext_completion"]
) -> Callable[..., Awaitable["TextCompletionResponse | TextCompletionStreamWrapper"]]: ...
@overload
def invoke(
self, op: Literal["aadapter_completion"]
) -> Callable[..., Awaitable["BaseModel | AdapterCompletionStreamWrapper | None"]]: ...
@overload
def invoke(self, op: Literal["aembedding"]) -> Callable[..., Awaitable["EmbeddingResponse"]]: ...
@overload
def invoke(self, op: Literal["aimage_generation"]) -> Callable[..., Awaitable["ImageResponse"]]: ...
@overload
def invoke(self, op: Literal["atranscription"]) -> Callable[..., Awaitable["TranscriptionResponse"]]: ...
@overload
def invoke(self, op: Literal["aspeech"]) -> Callable[..., Awaitable["HttpxBinaryResponseContent"]]: ...
@overload
def invoke(self, op: Literal["arerank"]) -> Callable[..., Awaitable["RerankResponse"]]: ...
@overload
def invoke(self, op: Literal["acreate_file"]) -> Callable[..., Awaitable["OpenAIFileObject"]]: ...
@overload
def invoke(self, op: Literal["acreate_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
@overload
def invoke(self, op: Literal["aretrieve_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
@overload
def invoke(self, op: Literal["acancel_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
@overload
def invoke(self, op: Literal["alist_batches"]) -> Callable[..., Awaitable[object]]: ...
@overload
def invoke(self, op: Literal["get_llm_provider"]) -> Callable[..., "tuple[str, str, str | None, str | None]"]: ...
@overload
def invoke(self, op: Literal["get_model_info"]) -> Callable[..., "ModelInfo"]: ...
@overload
def invoke(self, op: Literal["get_supported_openai_params"]) -> Callable[..., "Sequence[str] | None"]: ...
@overload
def invoke(self, op: Literal["token_counter"]) -> Callable[..., int]: ...
@overload
def invoke(self, op: Literal["calculate_retry_after"]) -> Callable[..., float]: ...
@overload
def invoke(self, op: str) -> Endpoint: ...
def invoke(self, op: str) -> Endpoint:
"""The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time."""
endpoint: Final[object] = getattr(litellm, op, None)
if endpoint is None:
raise AttributeError(f"litellm has no endpoint {op!r}")
if not callable(endpoint):
raise TypeError(f"litellm.{op} is not an endpoint")
return endpoint
def sleep(self, seconds: float) -> Awaitable[None]:
return asyncio.sleep(seconds)
def dynamic(self) -> RouterDynamicConfig:
"""The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot)."""
return cast(RouterDynamicConfig, live_dynamic_config) # cast-ok: the live view reads the same fields
@property
def callbacks(self) -> "LoggingCallbackManager":
return litellm.logging_callback_manager
def callback_list(self, name: CallbackList) -> list[object]: # mutable-ok: the process callback list itself
value: Final = getattr(litellm, name)
return cast(list[object], value) # cast-ok: the process callback list is an untyped litellm global
def set_callback_list(self, name: CallbackList, value: list[object]) -> None: # mutable-ok: assigned as given
setattr(litellm, name, value)
def global_cache(self) -> "Cache | None":
return litellm.cache
def set_global_cache(self, cache: "Cache | None") -> None:
litellm.cache = cache
def suppress_debug_info(self) -> None:
litellm.suppress_debug_info = True
python_host: Final = PythonHost()

View file

@ -1,15 +1,20 @@
import asyncio
import enum
import heapq
import logging as _logging
import time
from collections.abc import Awaitable, Callable, Sequence
from typing import Final, TypeAlias
from typing import Protocol as _Protocol
from pydantic import TypeAdapter
from litellm import print_verbose
from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger as _verbose_logger
from litellm.caching.caching import DualCache, RedisCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError as _RedisCircuitBreakerOpenError
from litellm.caching.redis_cache import log_redis_failure as _log_redis_failure
from litellm.constants import DEFAULT_IN_MEMORY_TTL, DEFAULT_POLLING_INTERVAL
from litellm.exceptions import Timeout
from litellm.types.llms.base import LiteLLMBaseModel
@ -30,6 +35,26 @@ class FlowItem(LiteLLMBaseModel):
model_name: str
class _QueueCache(_Protocol):
async def async_get_cache(self, key: str, *, local_only: bool = False, raise_on_error: bool = False) -> object: ...
async def async_set_cache(self, key: str, value: object, *, local_only: bool = False) -> object: ...
async def _read_queue(cache: _QueueCache, redis_cache: _QueueCache | None, key: str) -> object:
if redis_cache is None:
return await cache.async_get_cache(key=key, local_only=True)
from redis.exceptions import RedisError as _RedisError
try:
response: Final = await redis_cache.async_get_cache(key=key, raise_on_error=True)
except (_RedisError, _RedisCircuitBreakerOpenError) as error:
_log_redis_failure(_verbose_logger, _logging.ERROR, "Scheduler queue using local snapshot", error)
return await cache.async_get_cache(key=key, local_only=True)
await cache.async_set_cache(key=key, value=[] if response is None else response, local_only=True)
return response
class Scheduler:
cache: DualCache
@ -153,7 +178,7 @@ class Scheduler:
"""
if self.cache is not None:
_cache_key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}"
response: Final = await self.cache.async_get_cache(key=_cache_key)
response: Final = await _read_queue(self.cache, self.cache.redis_cache, _cache_key)
if not isinstance(response, list):
return []
return _QUEUE_ENTRIES.validate_python(response)

View file

@ -288,6 +288,7 @@ class PromptOptions:
@dataclass(frozen=True, slots=True, kw_only=True)
class ResponseOptions:
merge_reasoning_content_in_choices: bool | None = None
thought_signature_in_tool_call_id: bool | None = None
enable_json_schema_validation: bool | None = None
complete_response: bool | None = None
keepalive_seconds: float | None = None

View file

@ -751,7 +751,9 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel):
thinking_tokens: int | None = None
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"]
AnthropicFinishReason = Literal[
"end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "pause_turn", "compaction"
]
class AnthropicResponse(LiteLLMBaseModel):

View file

@ -98,7 +98,7 @@ class AnthropicMessagesResponse(TypedDict, total=False):
model: str | None # This represents the Model type from Anthropic
role: Literal["assistant"] | None
stop_reason: ReadOnly[
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction", "pause_turn"] | None
]
stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]]
stop_sequence: str | None

View file

@ -70,6 +70,7 @@ from pydantic import (
FieldSerializationInfo,
NonNegativeInt,
PrivateAttr,
SerializeAsAny,
SerializerFunctionWrapHandler,
field_serializer,
field_validator,
@ -1855,7 +1856,7 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
item_id: str
output_index: int
content_index: int
part: BaseLiteLLMOpenAIResponseObject
part: SerializeAsAny[BaseLiteLLMOpenAIResponseObject]
class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject):

View file

@ -4,6 +4,7 @@ from typing import Final, Literal, Optional, Union
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
from pydantic import ConfigDict, PrivateAttr
from typing_extensions import TypedDict
@ -154,7 +155,7 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject):
id: str
status: str # "completed", "in_progress", etc.
role: str # "assistant", "user", etc.
content: list[OutputText]
content: list[OutputText | ResponseOutputRefusal]
phase: Phase = None

View file

@ -25,7 +25,20 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import httpx
from openai import Timeout as SDKTimeout
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
JsonValue,
StrictInt,
ValidationError,
ValidationInfo,
ValidatorFunctionWrapHandler,
WrapValidator,
field_validator,
model_validator,
)
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
from litellm._logging import verbose_logger
@ -531,6 +544,27 @@ def holds_secret_pointer(param_name: str) -> bool:
_RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"})
_STRICT_LITELLM_PARAMS: Final = MappingProxyType({"strict_litellm_params": True})
_StrictlyValidatedParams: Final = TypeVar("_StrictlyValidatedParams", bound=BaseModel)
def validate_litellm_params_strictly(
params_model: type[_StrictlyValidatedParams], params: Mapping[str, object]
) -> _StrictlyValidatedParams:
return params_model.model_validate(params, context=_STRICT_LITELLM_PARAMS)
def _none_when_malformed_outside_strict_validation(
value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo
) -> object:
try:
return handler(value)
except ValidationError:
if info.context is _STRICT_LITELLM_PARAMS:
raise
return None
_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}")
@ -612,6 +646,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
)
model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
merge_reasoning_content_in_choices: bool | None = False
thought_signature_in_tool_call_id: Annotated[
bool | None, WrapValidator(_none_when_malformed_outside_strict_validation)
] = None
model_info: dict | None = None
mock_response: str | ModelResponse | Exception | object | None = None

View file

@ -677,13 +677,13 @@ class CallTypes(str, Enum):
retrieve_fine_tuning_job = "retrieve_fine_tuning_job"
responses = "responses"
aresponses = "aresponses"
alist_input_items = "alist_input_items"
get_responses = "get_responses"
aget_responses = "aget_responses"
delete_responses = "delete_responses"
adelete_responses = "adelete_responses"
cancel_responses = "cancel_responses"
acancel_responses = "acancel_responses"
alist_input_items = "alist_input_items"
compact_responses = "compact_responses"
acompact_responses = "acompact_responses"
llm_passthrough_route = "llm_passthrough_route"

View file

@ -1595,6 +1595,7 @@ def dispatch_success_logging(
is_litellm_internal_call: bool,
) -> None:
if is_litellm_internal_call:
logging_obj.capture_usage_before_logging(result, start_time, end_time)
return
_schedule_async_success_logging(
@ -1633,9 +1634,14 @@ def _schedule_async_success_logging(
if not logging_obj.claim_async_success_log():
return
if logging_obj.defer_async_logging and not is_completion_with_fallbacks:
logging_obj.capture_usage_before_logging(result, start_time, end_time, freeze_only=True)
def _enqueue_async_logging() -> None:
from litellm.responses.streaming_iterator import spawn_logging_task
if not is_completion_with_fallbacks:
logging_obj.capture_usage_before_logging(result, start_time, end_time)
if logging_obj.defer_async_logging:
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=result, start_time=start_time, end_time=end_time
@ -2203,6 +2209,7 @@ def client(original_function):
# LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated
verbose_logger.info("Wrapper: Completed Call, calling success_handler")
logging_obj.capture_usage_before_logging(TypeAdapter(object).validate_python(result), start_time, end_time)
if not is_internal_call.get():
# Copy the current context to propagate it to the background thread
# This is essential for OpenTelemetry span context propagation

View file

@ -81372,5 +81372,77 @@
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-6-astra-law": {
"cache_read_input_token_cost": 1e-06,
"input_cost_per_token": 1e-05,
"litellm_provider": "openai",
"max_input_tokens": 922000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-05,
"reasoning_contexts": [
"auto",
"current_turn",
"all_turns"
],
"reasoning_modes": [
"standard",
"pro"
],
"regional_processing_uplift_multiplier_eu": 1.1,
"regional_processing_uplift_multiplier_us": 1.1,
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_native_streaming": true,
"supports_none_reasoning_effort": false,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_cache_breakpoint": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_tool_search": true,
"supports_vision": true,
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"chatgpt/gpt-6-astra-law": {
"litellm_provider": "chatgpt",
"max_input_tokens": 922000,
"max_output_tokens": 128000,
"mode": "responses",
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_parallel_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
}
}

Some files were not shown because too many files have changed in this diff Show more