mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge remote-tracking branch 'origin/main' into litellm_responses_calltypes
# Conflicts: # ui/litellm-dashboard/src/lib/http/schema.d.ts
This commit is contained in:
commit
9da5899763
209 changed files with 36500 additions and 18482 deletions
7
.circleci/AGENTS.md
Normal file
7
.circleci/AGENTS.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# .circleci
|
||||
|
||||
The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
|
||||
|
||||
Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base
|
||||
|
||||
A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base
|
||||
|
|
@ -21,19 +21,30 @@ orbs:
|
|||
win: circleci/windows@5.0 # Add Windows orb
|
||||
|
||||
commands:
|
||||
checkout_migration_source:
|
||||
checkout_over_https:
|
||||
parameters:
|
||||
revision:
|
||||
type: string
|
||||
default: ""
|
||||
shell:
|
||||
type: string
|
||||
default: /bin/bash -eo pipefail
|
||||
steps:
|
||||
- run:
|
||||
name: Select the requested migration test revision
|
||||
name: Check out the public repository over HTTPS
|
||||
shell: << parameters.shell >>
|
||||
environment:
|
||||
MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
|
||||
REQUESTED_REVISION: << parameters.revision >>
|
||||
command: |
|
||||
revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
|
||||
revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
|
||||
[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
||||
git init
|
||||
git remote add origin https://github.com/BerriAI/litellm.git
|
||||
git fetch --depth 1 origin "$revision"
|
||||
git checkout --detach FETCH_HEAD
|
||||
for attempt in 1 2 3; do
|
||||
git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
|
||||
[[ $attempt -lt 3 ]] || exit 1
|
||||
sleep 5
|
||||
done
|
||||
skip_if_unrelated_changes:
|
||||
parameters:
|
||||
category:
|
||||
|
|
@ -347,7 +358,7 @@ commands:
|
|||
- when:
|
||||
condition: << parameters.checkout >>
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- install_rust
|
||||
|
|
@ -377,7 +388,8 @@ jobs:
|
|||
CARGO_HTTP_MULTIPLEXING: "false"
|
||||
CARGO_NET_RETRY: "5"
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https:
|
||||
shell: bash.exe
|
||||
- run:
|
||||
name: Skip job when no windows-release-relevant files changed
|
||||
shell: bash.exe
|
||||
|
|
@ -451,7 +463,7 @@ jobs:
|
|||
password: ${DOCKERHUB_PASSWORD}
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- setup_google_dns
|
||||
- install_uv
|
||||
- install_rust
|
||||
|
|
@ -482,7 +494,7 @@ jobs:
|
|||
working_directory: ~/project
|
||||
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
|
|
@ -559,7 +571,7 @@ jobs:
|
|||
resource_class: medium
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -659,7 +671,7 @@ jobs:
|
|||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -740,7 +752,7 @@ jobs:
|
|||
resource_class: large
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes:
|
||||
category: client
|
||||
- setup_google_dns
|
||||
|
|
@ -794,7 +806,7 @@ jobs:
|
|||
# the rest of the suite: nothing else performs a logout.
|
||||
PROXY_LOGOUT_URL: "https://www.example.com"
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -940,7 +952,7 @@ jobs:
|
|||
# and by migration.serverRootPath.config.ts, which refuses to run without it.
|
||||
SERVER_ROOT_PATH: "/litellm"
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -1066,11 +1078,12 @@ jobs:
|
|||
- when:
|
||||
condition: << parameters.migration_qualification >>
|
||||
steps:
|
||||
- checkout_migration_source
|
||||
- checkout_over_https:
|
||||
revision: << pipeline.parameters.migration_source_sha >>
|
||||
- unless:
|
||||
condition: << parameters.migration_qualification >>
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
|
||||
- run:
|
||||
|
|
@ -1122,7 +1135,8 @@ jobs:
|
|||
MIGRATION_TEST_OUTPUT: /tmp/migration-results
|
||||
PYTHONPATH: tests/e2e
|
||||
steps:
|
||||
- checkout_migration_source
|
||||
- checkout_over_https:
|
||||
revision: << pipeline.parameters.migration_source_sha >>
|
||||
- run:
|
||||
name: Generate LiteLLM master key
|
||||
command: |
|
||||
|
|
@ -1197,7 +1211,7 @@ jobs:
|
|||
resource_class: medium
|
||||
working_directory: ~/project
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
- attach_workspace:
|
||||
at: ~/project
|
||||
|
|
@ -1371,7 +1385,7 @@ jobs:
|
|||
environment:
|
||||
UV_PYTHON_DOWNLOADS: never
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- install_uv
|
||||
- run:
|
||||
name: Install the locked shard verifier and coverage dependencies
|
||||
|
|
@ -1419,7 +1433,7 @@ jobs:
|
|||
ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
|
||||
command: |
|
||||
[[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
|
||||
git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
|
||||
git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
|
||||
git rm -r -f --quiet litellm enterprise litellm-proxy-extras
|
||||
git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
|
||||
git reset --quiet
|
||||
|
|
@ -1494,7 +1508,7 @@ jobs:
|
|||
environment:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
|
||||
steps:
|
||||
- checkout
|
||||
- checkout_over_https
|
||||
- skip_if_unrelated_changes
|
||||
- setup_litellm_test_deps:
|
||||
checkout: false
|
||||
|
|
|
|||
9
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
9
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
|
|
@ -4,9 +4,14 @@
|
|||
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
|
||||
"## Affected release" with nothing under it must not appear in the final description -->
|
||||
|
||||
<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
|
||||
## Handwritten 1-3 sentence description
|
||||
|
||||
<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
|
||||
plain sentences proves the author understands what it does, and human-written technical English is far easier
|
||||
for a reviewer to parse than AI-written English
|
||||
An AI agent may draft a starting point here, but the author replaces it with their own handwritten version
|
||||
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
|
||||
No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
|
||||
No bullets, tables, or code names here. Everything else goes inside the Details block below -->
|
||||
|
||||
<details>
|
||||
<summary>Details</summary>
|
||||
|
|
|
|||
2
.github/workflows/create-rc-branch.yml
vendored
2
.github/workflows/create-rc-branch.yml
vendored
|
|
@ -2,7 +2,7 @@ name: Create RC Branch
|
|||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 3 * * 5"
|
||||
- cron: "0 3 * * 1"
|
||||
timezone: "America/Los_Angeles"
|
||||
workflow_dispatch:
|
||||
|
||||
|
|
|
|||
|
|
@ -200,6 +200,7 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
|
|||
unpriced: Final = _without_read_price(billed.response)
|
||||
cost: Final = logging_obj.response_cost_calculator(result=unpriced)
|
||||
end_time: Final = datetime.now()
|
||||
logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
async_coroutine=logging_obj.async_success_handler(
|
||||
result=unpriced, start_time=logging_obj.start_time, end_time=end_time
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
|
|||
("max_tokens", "length"),
|
||||
("refusal", "content_filter"),
|
||||
("compaction", "length"),
|
||||
("pause_turn", "length"),
|
||||
("model_context_window_exceeded", "length"),
|
||||
("guardrail_intervened", "content_filter"),
|
||||
("content_filtered", "content_filter"),
|
||||
|
|
@ -75,6 +76,7 @@ mod tests {
|
|||
#[case::end_turn("end_turn", "stop")]
|
||||
#[case::stop_sequence("stop_sequence", "stop")]
|
||||
#[case::max_tokens("max_tokens", "length")]
|
||||
#[case::pause_turn("pause_turn", "length")]
|
||||
#[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
|
||||
#[case::refusal("refusal", "content_filter")]
|
||||
#[case::guardrail_intervened("guardrail_intervened", "content_filter")]
|
||||
|
|
|
|||
1
litellm-rust/crates/cost/AGENTS.md
Normal file
1
litellm-rust/crates/cost/AGENTS.md
Normal file
|
|
@ -0,0 +1 @@
|
|||
The price vocabulary (meter, band, channel, `When`, `Billing`) is explained in [../model-catalog/AGENTS.md](../model-catalog/AGENTS.md#concepts), the only crate that maps it to JSON keys
|
||||
|
|
@ -19,6 +19,7 @@ pub enum ServiceTier {
|
|||
Balanced,
|
||||
}
|
||||
|
||||
/// A prompt-size range. `OverPromptTokens(n)` is spelled `_above_<n>_tokens` or `_above_<n / 1000>k_tokens`
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum Band {
|
||||
#[default]
|
||||
|
|
@ -26,12 +27,15 @@ pub enum Band {
|
|||
OverPromptTokens(NonZeroU64),
|
||||
}
|
||||
|
||||
/// The request condition a price applies under, its service tier and prompt band
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct When {
|
||||
pub tier: ServiceTier,
|
||||
pub band: Band,
|
||||
}
|
||||
|
||||
/// The route whose price list a call reads, from the `_batches` and `_decisions` suffixes. Not part of `When`,
|
||||
/// since a missing batch price is half the online rate and a missing decisions price is the full online rate
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum Channel {
|
||||
#[default]
|
||||
|
|
@ -40,6 +44,7 @@ pub enum Channel {
|
|||
Decisions,
|
||||
}
|
||||
|
||||
/// The key of one rate, what is counted, through which channel and under which condition
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct Price {
|
||||
pub meter: Meter,
|
||||
|
|
@ -47,6 +52,7 @@ pub struct Price {
|
|||
pub when: When,
|
||||
}
|
||||
|
||||
/// Every rate a row lists, its off-peak overlay and the row attributes that say how the rates apply
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct Prices {
|
||||
rates: BTreeMap<Price, UnitPrice>,
|
||||
|
|
@ -117,6 +123,7 @@ pub enum TierScope {
|
|||
WholeCall,
|
||||
}
|
||||
|
||||
/// Rates that replace the row's token rates while a UTC window in the schedule is open
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OffPeak {
|
||||
schedule: Schedule,
|
||||
|
|
@ -133,6 +140,7 @@ pub struct OffPeakRates {
|
|||
pub reasoning: Option<UnitPrice>,
|
||||
}
|
||||
|
||||
/// Multipliers applied on top of a priced call, never prices themselves
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct Adjustments(BTreeMap<Uplift, Multiplier>);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::num::NonZeroU64;
|
||||
|
||||
/// What a price counts. A variation only one meter has, such as cache TTL or image size, lives inside its variant
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum Meter {
|
||||
Input(InputUnit),
|
||||
|
|
@ -51,12 +52,14 @@ pub enum OutputUnit {
|
|||
VideoSecond,
|
||||
}
|
||||
|
||||
/// `Explicit` is a cache the caller created ahead of time, spelled `explicit_cache_read_*`. `Implicit` is every other read
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum CacheReadMode {
|
||||
Implicit,
|
||||
Explicit,
|
||||
}
|
||||
|
||||
/// How long a cache write lives. `OneHour` and `TwelveHours` are spelled `_above_1hr` and `_above_12hr`
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum CacheTtl {
|
||||
Default,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,41 @@
|
|||
## Concepts
|
||||
|
||||
Every price in a row lowers to one `Price { meter, channel, when }` and its `UnitPrice`. These keys come from the `FlatKey::parse` tests
|
||||
|
||||
- `input_cost_per_token`
|
||||
- meter `Input(Token(Text))`, channel `Online`, tier `Standard`, band `Base`
|
||||
- `output_cost_per_token_above_272k_tokens_priority`
|
||||
- meter `Output(Token(Text))`, channel `Online`
|
||||
- `_priority` sets tier `Priority`
|
||||
- `_above_272k_tokens` sets band `OverPromptTokens(272000)`
|
||||
- `cache_creation_input_token_cost_above_1hr_above_200k_tokens`
|
||||
- meter `CacheWrite { Text, OneHour }`, where `_above_1hr` is the cache TTL, not a band
|
||||
- `_above_200k_tokens` sets band `OverPromptTokens(200000)`
|
||||
- channel `Online`, tier `Standard`
|
||||
- `ocr_cost_per_page_batches`
|
||||
- meter `Feature(OcrPage)`, tier `Standard`, band `Base`
|
||||
- `_batches` sets channel `Batch`
|
||||
|
||||
- Meter is what gets counted, such as text input tokens, reasoning tokens, an image of a given size or a web search
|
||||
- A variation only one meter has stays inside it (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`)
|
||||
- `UnitPrice` is USD for one unit of the meter, such as one token, image, page or second
|
||||
- Band is a range of prompt sizes, starting where `_above_<N>_tokens` or `_above_<N>k_tokens` says
|
||||
- `Prices::band` picks one band from prompt tokens alone, the highest edge passed wins, and `Band::Base` is below every edge
|
||||
- The whole call moves, so every token meter, output included, bills all its tokens at that band's rate
|
||||
- `Billing::band_edge` says whether a prompt exactly at the edge is in the band (exclusive by default, inclusive for xAI)
|
||||
- A band only another tier or the batch channel prices never moves the call
|
||||
- `When` is the request's condition, its service tier and band
|
||||
- A missing price falls back from `(tier, band)` to `(standard, band)`, `(tier, base)`, then `(standard, base)`
|
||||
- `Channel` is the route whose price list the call reads, and each falls back differently
|
||||
- `Online` falls back across `When` as above
|
||||
- `Batch` (`_batches`) uses bands but no service tier, and a missing price is half the online rate
|
||||
- `Decisions` (`_decisions`, OpenAI `/v1/decisions`) overrides single online fields, and a missing price is the full online rate
|
||||
- Off-peak and `Adjustments` are never prices
|
||||
- Off-peak swaps in other token rates while a UTC window is open
|
||||
- `Adjustments` multiplies a priced call for region, residency, geo or `speed: fast`
|
||||
- `Billing` on `Prices` holds row attributes that change how its prices apply (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope)
|
||||
- `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
|
||||
|
||||
## Layout
|
||||
|
||||
`wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read
|
||||
|
|
@ -6,8 +44,6 @@
|
|||
|
||||
`catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations`
|
||||
|
||||
`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
|
||||
|
||||
`store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md)
|
||||
|
||||
`layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller
|
||||
|
|
@ -16,7 +52,7 @@ Catalog errors use one opaque `Error` with typed causes and JSON Pointer accesso
|
|||
|
||||
## Prices
|
||||
|
||||
The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices
|
||||
The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names
|
||||
|
||||
A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs`
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ _T = TypeVar("_T")
|
|||
# When True, suppresses async logging and billing for internal sub-calls
|
||||
# (e.g., emulated file-search steps that make nested LLM calls).
|
||||
is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
|
||||
internal_parent_request_id: Final[ContextVar[str | None]] = ContextVar("internal_parent_request_id", default=None)
|
||||
|
||||
# One request prices its totals, its per-token-type lines and the rates it reports on
|
||||
# separate code paths. Each reads the clock for off-peak pricing, so without a pinned
|
||||
|
|
@ -40,13 +41,15 @@ _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=Fa
|
|||
|
||||
|
||||
@contextmanager
|
||||
def internal_sub_call() -> Generator[None]:
|
||||
def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
|
||||
"""A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
|
||||
token: Final = is_internal_call.set(True)
|
||||
parent_token: Final = internal_parent_request_id.set(parent_request_id or internal_parent_request_id.get())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
is_internal_call.reset(token)
|
||||
internal_parent_request_id.reset(parent_token)
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
|
||||
Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
|
||||
model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
|
||||
These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
|
||||
These inputs, including the effective cost-map rates, must survive request teardown.
|
||||
Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
|
||||
T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
|
||||
"""
|
||||
|
|
@ -38,8 +38,8 @@ class CostCompiler(Protocol):
|
|||
async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
|
||||
"""Reuse the committed charge or finalize one charge and its delivery intent.
|
||||
|
||||
Resolve captured request-scoped context by event identity and cost-map rates
|
||||
at compilation. Persist the resolved inputs in the required snapshot. Replay
|
||||
Resolve captured request-scoped context and rates by event identity.
|
||||
Persist the resolved inputs in the required snapshot. Replay
|
||||
reuses the committed result and never consumes free allowance again.
|
||||
Missing pricing is Unpriced, never an inferred zero.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -276,8 +276,8 @@ class NamedCost(BaseModel):
|
|||
class PricingSnapshot(BaseModel):
|
||||
"""Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
|
||||
|
||||
Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
|
||||
resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
|
||||
Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
|
||||
Retries reuse the committed charge and snapshot.
|
||||
Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
|
||||
Component sums and float tolerances belong to reconciliation, not this structural contract.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|||
from dataclasses import dataclass
|
||||
from dataclasses import replace as dataclasses_replace
|
||||
from enum import Enum
|
||||
from itertools import groupby
|
||||
from typing import Any, Final, Literal, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -15,6 +16,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
|
|||
from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
|
||||
from litellm.llms.vertex_ai.batches.transformation import (
|
||||
is_native_vertex_batch_output_row,
|
||||
native_vertex_batch_row_model,
|
||||
native_vertex_batch_row_stats,
|
||||
)
|
||||
from litellm.types.llms.openai import Batch
|
||||
|
|
@ -210,6 +212,7 @@ class _BatchOutputLineStats:
|
|||
cache_creation_tokens: int
|
||||
reasoning_tokens: int
|
||||
model: str | None
|
||||
usage: Usage | None
|
||||
|
||||
|
||||
def _classify_output_line_stats(
|
||||
|
|
@ -268,7 +271,7 @@ def _compute_output_line_stats(
|
|||
raw_model: Final = response_body.get("model")
|
||||
response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
|
||||
completion_details: Final = usage.completion_tokens_details
|
||||
line_prompt_cost, line_completion_cost = _output_line_cost(
|
||||
line_prompt_cost, line_completion_cost = _output_line_cost_or_pending(
|
||||
response_body=response_body,
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -286,7 +289,15 @@ def _compute_output_line_stats(
|
|||
cache_read_tokens=prompt_details["cache_hit_tokens"],
|
||||
cache_creation_tokens=prompt_details["cache_creation_tokens"],
|
||||
reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
|
||||
model=response_model,
|
||||
model=response_model or model_name,
|
||||
usage=(
|
||||
usage.model_copy(update=ocr_usage.model_dump(exclude_none=True))
|
||||
if (ocr_usage := _ocr_usage_info_from_response_body(response_body)) is not None
|
||||
and ocr_usage.model_dump(exclude_none=True)
|
||||
else usage
|
||||
if response_body.get("usage") is not None or usage.total_tokens > 0
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -298,6 +309,54 @@ def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> O
|
|||
return OCRUsageInfo.model_validate(raw_usage_info)
|
||||
|
||||
|
||||
def _output_line_cost_or_pending(
|
||||
response_body: Mapping[str, object],
|
||||
usage: Usage,
|
||||
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
|
||||
model_name: str | None,
|
||||
response_model: str | None,
|
||||
model_info: ModelInfo | None,
|
||||
vertex_location: str | None,
|
||||
) -> tuple[float, float]:
|
||||
try:
|
||||
return _output_line_cost(
|
||||
response_body, usage, custom_llm_provider, model_name, response_model, model_info, vertex_location
|
||||
)
|
||||
except Exception: # noqa: BLE001 # retain measured usage when existing batch pricing fails
|
||||
verbose_logger.warning("Batch line pricing unavailable; retaining usage with the existing zero-cost fallback")
|
||||
return 0.0, 0.0
|
||||
|
||||
|
||||
def _combined_reported_batch_usage(usages: tuple[Usage | None, ...]) -> Mapping[str, object] | None:
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor
|
||||
|
||||
reported: Final = [usage for usage in usages if usage is not None]
|
||||
if not reported:
|
||||
return None
|
||||
fields: Final = tuple(TypeAdapter(dict[str, JsonValue]).validate_python(usage.model_dump()) for usage in reported)
|
||||
quantities: Final = {
|
||||
key: sum(value for usage in fields if isinstance(value := usage.get(key), (int, float)))
|
||||
for key in OCRUsageInfo.model_fields
|
||||
if any(isinstance(usage.get(key), (int, float)) for usage in fields)
|
||||
}
|
||||
return {**BaseTokenUsageProcessor.combine_usage_objects(usage_objects=reported).model_dump(), **quantities}
|
||||
|
||||
|
||||
def _batch_usage_by_request(usages: tuple[tuple[str | None, Usage | None], ...]) -> tuple[Mapping[str, object], ...]:
|
||||
rows: Final = tuple(
|
||||
{"model": model, "usage": usage.model_dump() if usage is not None else None} for model, usage in usages
|
||||
)
|
||||
return tuple(sorted(rows, key=lambda row: json.dumps(row, sort_keys=True)))
|
||||
|
||||
|
||||
def _batch_usage_by_model(usages: tuple[tuple[str | None, Usage | None], ...]) -> Mapping[str, object]:
|
||||
ordered: Final = sorted(usages, key=lambda item: item[0] or "")
|
||||
return {
|
||||
model: _combined_reported_batch_usage(tuple(item[1] for item in group))
|
||||
for model, group in groupby(ordered, key=lambda item: item[0] or "")
|
||||
}
|
||||
|
||||
|
||||
def _output_line_cost(
|
||||
response_body: Mapping[str, object],
|
||||
usage: Usage,
|
||||
|
|
@ -361,6 +420,9 @@ def _aggregate_batch_cost_usage_models(
|
|||
completion_tokens=sum(stats.completion_tokens for stats in line_stats),
|
||||
reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
|
||||
**cache_token_params,
|
||||
batch_usage_by_model=_batch_usage_by_model(tuple((stats.model, stats.usage) for stats in line_stats)),
|
||||
batch_usage_by_request=_batch_usage_by_request(tuple((stats.model, stats.usage) for stats in line_stats)),
|
||||
batch_unknown_usage_requests=sum(stats.usage is None for stats in line_stats),
|
||||
)
|
||||
batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
|
||||
total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0)
|
||||
|
|
@ -401,17 +463,21 @@ def calculate_vertex_ai_batch_cost_and_usage(
|
|||
from litellm.cost_calculator import batch_cost_calculator
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
|
||||
|
||||
row_stats: Final = tuple(
|
||||
native_vertex_batch_row_stats(
|
||||
row,
|
||||
model_name,
|
||||
model_info=model_info,
|
||||
calculate_usage=VertexGeminiConfig.calculate_usage,
|
||||
cost_calculator=batch_cost_calculator,
|
||||
vertex_location=vertex_location,
|
||||
observations: Final = tuple(
|
||||
(
|
||||
native_vertex_batch_row_model(row, model_name),
|
||||
native_vertex_batch_row_stats(
|
||||
row,
|
||||
model_name,
|
||||
model_info=model_info,
|
||||
calculate_usage=VertexGeminiConfig.calculate_usage,
|
||||
cost_calculator=batch_cost_calculator,
|
||||
vertex_location=vertex_location,
|
||||
),
|
||||
)
|
||||
for row in vertex_ai_batch_responses
|
||||
)
|
||||
row_stats: Final = tuple(stats for _, stats in observations)
|
||||
priced: Final = tuple(stats for stats in row_stats if stats is not None)
|
||||
total_prompt_cost: Final = sum(stats.prompt_cost for stats in priced)
|
||||
total_completion_cost: Final = sum(stats.completion_cost for stats in priced)
|
||||
|
|
@ -435,6 +501,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
|
|||
total_tokens=total_tokens,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
batch_usage_by_model=_batch_usage_by_model(
|
||||
tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
|
||||
),
|
||||
batch_usage_by_request=_batch_usage_by_request(
|
||||
tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
|
||||
),
|
||||
batch_unknown_usage_requests=sum(stats is None for stats in row_stats),
|
||||
),
|
||||
models=(
|
||||
[model_name]
|
||||
|
|
|
|||
|
|
@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
|
||||
from litellm.integrations.otel.runtime import phase_span
|
||||
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
|
||||
from litellm.litellm_core_utils.model_param_helper import (
|
||||
ModelParamHelper,
|
||||
embeds_thought_signature_in_tool_call_id,
|
||||
peek_reasoning_summary_aliases,
|
||||
)
|
||||
from litellm.types.caching import *
|
||||
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
|
||||
|
||||
|
|
@ -421,7 +425,12 @@ class Cache:
|
|||
reasoning_summary_material: Final = (
|
||||
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
|
||||
)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
|
||||
tool_call_id_shape_material: Final = (
|
||||
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
|
||||
)
|
||||
hashed_cache_key = Cache._get_hashed_cache_key(
|
||||
cache_key + reasoning_summary_material + tool_call_id_shape_material
|
||||
)
|
||||
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
|
||||
verbose_logger.debug(
|
||||
"\nCreated cache key: %s (source material length: %d)",
|
||||
|
|
|
|||
|
|
@ -432,6 +432,7 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
|
|||
DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
|
||||
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
|
||||
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
|
||||
DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en"
|
||||
|
||||
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
|
||||
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
|
||||
|
|
@ -675,7 +676,7 @@ request_timeout: float = (
|
|||
DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env
|
||||
)
|
||||
request_timeout_explicitly_set: bool = _request_timeout_from_env is not None
|
||||
DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0) # 10 minutes
|
||||
DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)
|
||||
AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0
|
||||
AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000
|
||||
# Patterns that indicate a localhost/internal URL in A2A agent cards that should be
|
||||
|
|
@ -1691,6 +1692,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
|
|||
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
|
||||
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
|
||||
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
|
||||
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
|
||||
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
|
||||
|
|
|
|||
|
|
@ -1554,12 +1554,66 @@ def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float
|
|||
return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
|
||||
|
||||
|
||||
def _cost_map_priced_input_output(
|
||||
model: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
usage: Usage | None,
|
||||
call_type: CallTypesLiteral | None,
|
||||
service_tier: str | None,
|
||||
data_residency: str | None,
|
||||
) -> tuple[float, float]:
|
||||
if not model or usage is None:
|
||||
return 0.0, 0.0
|
||||
try:
|
||||
return cost_per_token(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage_object=usage,
|
||||
call_type=call_type or "completion",
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
except Exception as pricing_error: # noqa: BLE001 # cost_per_token raises plain Exception for an unmapped model
|
||||
verbose_logger.debug("No cost map price for reported-cost split of %s: %s", model, pricing_error)
|
||||
return 0.0, 0.0
|
||||
|
||||
|
||||
def split_reported_cost_by_cost_map_ratio(
|
||||
reported_total: float,
|
||||
model: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
usage: Usage | None,
|
||||
call_type: CallTypesLiteral | None,
|
||||
service_tier: str | None,
|
||||
data_residency: str | None,
|
||||
) -> tuple[float, float]:
|
||||
"""Apportions a provider-reported total between input and output in the ratio litellm's own cost map would
|
||||
price the same usage, so a per-input-token model books the whole total on input. Unmapped or zero-priced
|
||||
usage keeps the whole total on output."""
|
||||
priced_input, priced_output = _cost_map_priced_input_output(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
call_type=call_type,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
priced_total: Final = priced_input + priced_output
|
||||
if priced_total <= 0:
|
||||
return 0.0, reported_total
|
||||
input_cost: Final = reported_total * (priced_input / priced_total)
|
||||
return input_cost, reported_total - input_cost
|
||||
|
||||
|
||||
def _adjusted_provider_billed_cost(
|
||||
reported_cost: float,
|
||||
custom_llm_provider: str | None,
|
||||
pricing_provider: str | None,
|
||||
model: str | None,
|
||||
region_name: str | None,
|
||||
litellm_logging_obj: LitellmLoggingObject | None,
|
||||
usage: Usage | None,
|
||||
call_type: CallTypesLiteral | None,
|
||||
service_tier: str | None,
|
||||
data_residency: str | None,
|
||||
) -> float:
|
||||
|
|
@ -1569,10 +1623,19 @@ def _adjusted_provider_billed_cost(
|
|||
adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
|
||||
base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
input_cost, output_cost = split_reported_cost_by_cost_map_ratio(
|
||||
reported_total=adjusted_cost,
|
||||
model=model,
|
||||
custom_llm_provider=pricing_provider,
|
||||
usage=usage,
|
||||
call_type=call_type,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=adjusted_cost,
|
||||
prompt_tokens_cost_usd_dollar=input_cost,
|
||||
completion_tokens_cost_usd_dollar=output_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=adjusted_cost,
|
||||
original_cost=reported_cost,
|
||||
|
|
@ -2080,9 +2143,16 @@ def completion_cost(
|
|||
return _adjusted_provider_billed_cost(
|
||||
reported_cost=reported_cost,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
pricing_provider=_pricing_provider_for_cost_candidate(
|
||||
candidate_provider=candidate_pricing_provider,
|
||||
request_provider=request_pricing_provider,
|
||||
current_provider=custom_llm_provider,
|
||||
),
|
||||
model=model_for_discount,
|
||||
region_name=region_name if isinstance(region_name, str) else None,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
usage=cost_per_token_usage_object,
|
||||
call_type=call_type,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
|
|
@ -2387,10 +2457,28 @@ def response_cost_calculator(
|
|||
provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
|
||||
if provider_response_cost is not None and not custom_pricing:
|
||||
if _breakdown_total(litellm_logging_obj) != provider_response_cost:
|
||||
header_priced_model: Final = select_model_name_for_cost_calc(
|
||||
model=model,
|
||||
completion_response=response_object,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
router_model_id=router_model_id,
|
||||
region_name=region_name,
|
||||
)
|
||||
header_input_cost, header_output_cost = split_reported_cost_by_cost_map_ratio(
|
||||
reported_total=provider_response_cost,
|
||||
model=header_priced_model or model,
|
||||
custom_llm_provider=pricing_provider or custom_llm_provider,
|
||||
usage=get_usage_object(response_object),
|
||||
call_type=call_type,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
_store_cost_breakdown_in_logging_obj(
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
prompt_tokens_cost_usd_dollar=0.0,
|
||||
completion_tokens_cost_usd_dollar=provider_response_cost,
|
||||
prompt_tokens_cost_usd_dollar=header_input_cost,
|
||||
completion_tokens_cost_usd_dollar=header_output_cost,
|
||||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
total_cost_usd_dollar=provider_response_cost,
|
||||
original_cost=provider_response_cost,
|
||||
|
|
@ -3319,7 +3407,7 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
|
|||
)
|
||||
|
||||
|
||||
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
|
||||
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
|
||||
|
||||
|
||||
class _ResponsesWsEventResponse(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ class SpeechToCompletionBridgeHandler:
|
|||
voice=voice,
|
||||
)
|
||||
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(logging_obj.litellm_call_id):
|
||||
result: Final = completion(
|
||||
**request_data,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import copy
|
|||
import itertools
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
|
||||
from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
|
@ -445,22 +445,41 @@ def _as_object_list(value: object | None) -> list[object] | None:
|
|||
return _validated_object_list(value)
|
||||
|
||||
|
||||
def _tool_call_carries_cache_breakpoint(tool_call: object, message: object, answered_tool_ids: Container[str]) -> bool:
|
||||
def _server_tool_call_id(tool_call: object) -> str | None:
|
||||
tool_call_id: Final = _attribute_or_key(tool_call, "id")
|
||||
if isinstance(tool_call_id, str) and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
|
||||
return tool_call_id
|
||||
return None
|
||||
|
||||
|
||||
def _tool_call_carries_cache_breakpoint(tool_call: object, client_tool_call_ids: Container[str]) -> bool:
|
||||
if _attribute_or_key(tool_call, "cache_control") is None:
|
||||
return False
|
||||
server_tool_call_id: Final = _server_tool_call_id(tool_call)
|
||||
return server_tool_call_id is None or server_tool_call_id in client_tool_call_ids
|
||||
|
||||
return not tool_call_is_rebuilt_as_server_tool_use(
|
||||
_attribute_or_key(tool_call, "id"), _attribute_or_key(message, "provider_specific_fields"), answered_tool_ids
|
||||
|
||||
def _server_tool_call_ids_without_a_kept_result(message: object) -> Iterator[str]:
|
||||
provider_specific_fields: Final = _attribute_or_key(message, "provider_specific_fields")
|
||||
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or ()
|
||||
return (
|
||||
tool_call_id
|
||||
for tool_call in tool_calls
|
||||
if (tool_call_id := _server_tool_call_id(tool_call)) is not None
|
||||
and not tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
|
||||
)
|
||||
|
||||
|
||||
def _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
|
||||
def _client_tool_call_ids_when_a_server_tool_call_is_marked(
|
||||
messages: Sequence[object], model: str | None = None, custom_llm_provider: str | None = None
|
||||
) -> frozenset[str]:
|
||||
if not _marks_a_server_tool_call(messages):
|
||||
return frozenset()
|
||||
if model is not None and not _builds_anthropic_messages(model, custom_llm_provider):
|
||||
return tool_call_ids_answered_in(messages)
|
||||
server_calls_sent_as_client_calls: Final = itertools.chain.from_iterable(
|
||||
map(_server_tool_call_ids_without_a_kept_result, messages)
|
||||
)
|
||||
return tool_call_ids_answered_in(messages) | frozenset(server_calls_sent_as_client_calls)
|
||||
return tool_call_ids_answered_by_client(messages, _keeps_mid_conversation_system(model, custom_llm_provider))
|
||||
|
||||
|
||||
|
|
@ -517,12 +536,7 @@ def _model_info_flags_mid_conversation_system(model: str, custom_llm_provider: s
|
|||
|
||||
|
||||
def _is_marked_server_tool_call(tool_call: object) -> bool:
|
||||
tool_call_id: Final = _attribute_or_key(tool_call, "id")
|
||||
return (
|
||||
isinstance(tool_call_id, str)
|
||||
and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
|
||||
and _attribute_or_key(tool_call, "cache_control") is not None
|
||||
)
|
||||
return _server_tool_call_id(tool_call) is not None and _attribute_or_key(tool_call, "cache_control") is not None
|
||||
|
||||
|
||||
def _tool_carries_cache_breakpoint(tool: object) -> bool:
|
||||
|
|
@ -668,7 +682,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
|
||||
remaining_points, external_breakpoints, openai_dialect
|
||||
)
|
||||
answered_tool_ids: Final = _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
|
||||
answered_tool_ids: Final = _client_tool_call_ids_when_a_server_tool_call_is_marked(
|
||||
processed_messages, model, custom_llm_provider
|
||||
)
|
||||
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(
|
||||
|
|
@ -757,7 +771,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
)
|
||||
message_list: Final = tuple(messages)
|
||||
client_answers: Final = (
|
||||
_tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(message_list)
|
||||
_client_tool_call_ids_when_a_server_tool_call_is_marked(message_list)
|
||||
if answered_tool_ids is None
|
||||
else answered_tool_ids
|
||||
)
|
||||
|
|
@ -849,7 +863,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
so earlier points win when slots are scarce.
|
||||
"""
|
||||
client_answers: Final = (
|
||||
_tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(messages)
|
||||
_client_tool_call_ids_when_a_server_tool_call_is_marked(messages)
|
||||
if answered_tool_ids is None
|
||||
else answered_tool_ids
|
||||
)
|
||||
|
|
@ -939,11 +953,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
|
||||
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
|
||||
tool_call_count: Final = (
|
||||
sum(
|
||||
1
|
||||
for tool_call in tool_calls
|
||||
if _tool_call_carries_cache_breakpoint(tool_call, message, answered_tool_ids)
|
||||
)
|
||||
sum(1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, answered_tool_ids))
|
||||
if tool_calls
|
||||
else 0
|
||||
)
|
||||
|
|
|
|||
|
|
@ -311,6 +311,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
|
|||
"tool_use": "tool_calls",
|
||||
"refusal": "content_filter",
|
||||
"compaction": "length",
|
||||
"pause_turn": "length",
|
||||
"model_context_window_exceeded": "length",
|
||||
# Cohere
|
||||
"COMPLETE": "stop",
|
||||
|
|
@ -1073,9 +1074,14 @@ def billed_usage_cost(response: object, custom_llm_provider: str | None) -> floa
|
|||
_MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
|
||||
|
||||
|
||||
def reports_native_messages_cost(custom_llm_provider: object) -> bool:
|
||||
"""Whether a Messages upstream is one whose own reported cost litellm bills from instead of the price map."""
|
||||
return isinstance(custom_llm_provider, str) and custom_llm_provider in _MESSAGES_REPORTED_COST_PROVIDERS
|
||||
|
||||
|
||||
def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None:
|
||||
"""Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map."""
|
||||
if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS:
|
||||
if not isinstance(custom_llm_provider, str) or not reports_native_messages_cost(custom_llm_provider):
|
||||
return None
|
||||
return provider_reported_usage_cost(usage, custom_llm_provider)
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"github_copilot_host",
|
||||
"github_copilot_user_session",
|
||||
"fireworks_forward_user_id",
|
||||
"thought_signature_in_tool_call_id",
|
||||
"responses_api_version",
|
||||
"allowed_openai_params",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
|
|
@ -99,6 +100,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
# Backward-compatible alias for existing imports/tests.
|
||||
_OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
|
||||
|
||||
SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"}
|
||||
|
||||
_CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
|
||||
_CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
|
||||
_MAX_SHOWN_INT_BITS: Final = 64
|
||||
|
|
|
|||
|
|
@ -16,9 +16,10 @@ import traceback
|
|||
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime as dt_object
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType, TracebackType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
|
||||
|
||||
from httpx import Response, Timeout
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
|
@ -47,6 +48,7 @@ from litellm.constants import (
|
|||
KUBERNETES_POD_ROUTING_KEY,
|
||||
LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
|
||||
PROVIDER_REQUEST_ID_HEADERS,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REDACTED_BY_LITELLM,
|
||||
)
|
||||
from litellm.cost_calculator import (
|
||||
|
|
@ -84,6 +86,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
proxy_stamped_kubernetes_pod_routing,
|
||||
proxy_stamped_used_client_oauth_token,
|
||||
reconstruct_model_name,
|
||||
reports_native_messages_cost,
|
||||
set_response_cost_in_hidden_params,
|
||||
single_log_line,
|
||||
)
|
||||
|
|
@ -163,6 +166,7 @@ from litellm.types.llms.openai import (
|
|||
ResponseIncompleteEvent,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
from litellm.types.llms.vertex_ai import GenerateContentResponseBody, UsageMetadata
|
||||
from litellm.types.mcp import MCPPostCallResponseObject
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.rerank import RerankResponse, get_rerank_usage
|
||||
|
|
@ -266,6 +270,9 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
|
||||
from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
|
||||
from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
|
||||
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
|
||||
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
|
||||
try:
|
||||
from litellm_enterprise.enterprise_callbacks.callback_controls import (
|
||||
EnterpriseCallbackControls,
|
||||
|
|
@ -571,6 +578,7 @@ def _resolve_mantle_region_for_cost(
|
|||
_PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
|
||||
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
|
||||
_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
|
||||
_REDACTED_RESULT: Final = TypeAdapter(object)
|
||||
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
|
||||
|
||||
|
|
@ -630,7 +638,60 @@ def same_attempt_marker(first: object, second: object) -> bool:
|
|||
return first is second or first == second
|
||||
|
||||
|
||||
class UsageObservation(Enum):
|
||||
KNOWN = "known"
|
||||
UNKNOWN = "unknown"
|
||||
ESTIMATED = "estimated"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UsageCaptureObservation:
|
||||
usage_json: str | None
|
||||
provenance: UsageObservation
|
||||
response_cost: float | None = None
|
||||
|
||||
@classmethod
|
||||
def from_usage(
|
||||
cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None
|
||||
) -> "UsageCaptureObservation | UsageCaptureError":
|
||||
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
|
||||
|
||||
try:
|
||||
serialized: Final = (
|
||||
None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json()
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # observation failures must not interrupt legacy retry handling
|
||||
return UsageCaptureError(str(error))
|
||||
else:
|
||||
return cls(serialized, provenance, response_cost)
|
||||
|
||||
|
||||
class Logging(LiteLLMLoggingBaseClass):
|
||||
usage_capture_callback: ClassVar[
|
||||
Callable[
|
||||
["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None"
|
||||
]
|
||||
| None
|
||||
] = None
|
||||
|
||||
def capture_usage_before_logging(
|
||||
self,
|
||||
response: object,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
*,
|
||||
freeze_only: bool = False,
|
||||
) -> None:
|
||||
capture: Final = type(self).usage_capture_callback
|
||||
if capture is not None and (not self.stream or self._is_assembled_stream_success(response)):
|
||||
capture(
|
||||
self,
|
||||
response,
|
||||
start_time or self.start_time,
|
||||
end_time or datetime.datetime.now(datetime.timezone.utc),
|
||||
freeze_only,
|
||||
)
|
||||
|
||||
global \
|
||||
supabaseClient, \
|
||||
promptLayerLogger, \
|
||||
|
|
@ -828,6 +889,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.finalized_cost: FinalizedCost | None = None
|
||||
self.prepared_cost: PreparedCost | None = None
|
||||
self.cost_allowance_discount: float | None = None
|
||||
self.cost_allowance_allocation: SearchAllowanceAllocation | None = None
|
||||
self.cost_finalization_lock = asyncio.Lock()
|
||||
self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
|
||||
self.billed_token_rates: BilledTokenRates | None = None
|
||||
|
|
@ -884,7 +946,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
new_call.start_time = start_time
|
||||
new_call.litellm_params = litellm_params
|
||||
new_call.model_call_details = {
|
||||
**self.model_call_details,
|
||||
**{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"start_time": start_time,
|
||||
"first_api_call_start_time": None,
|
||||
|
|
@ -894,6 +956,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
new_call.finalized_cost = None
|
||||
new_call.prepared_cost = None
|
||||
new_call.cost_allowance_discount = None
|
||||
new_call.cost_allowance_allocation = None
|
||||
new_call.cost_finalization_lock = asyncio.Lock()
|
||||
new_call.deferred_success_result = None
|
||||
new_call.streaming_chunks = []
|
||||
|
|
@ -2424,7 +2487,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
_is_sync_litellm_request = is_sync_litellm_request
|
||||
|
||||
def _is_assembled_stream_success(self, result=None) -> bool:
|
||||
def _is_assembled_stream_success(self, result: object = None) -> bool:
|
||||
"""Final assembled stream export (not a per-chunk success call).
|
||||
|
||||
Per-chunk callers pass a ``ModelResponseStream`` (or ``None``); the
|
||||
|
|
@ -2459,7 +2522,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"""
|
||||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
|
||||
if self._is_assembled_stream_success(result):
|
||||
if self._is_assembled_stream_success(TypeAdapter(object).validate_python(result)):
|
||||
if self.model_call_details.get("has_dispatched_final_stream_success"):
|
||||
return
|
||||
self.model_call_details["has_dispatched_final_stream_success"] = True
|
||||
|
|
@ -2477,6 +2540,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
return
|
||||
|
||||
self.capture_usage_before_logging(
|
||||
TypeAdapter(object).validate_python(result),
|
||||
TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time),
|
||||
TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time),
|
||||
)
|
||||
await self.async_success_handler(
|
||||
result,
|
||||
start_time=start_time,
|
||||
|
|
@ -2503,6 +2571,17 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return self.prepared_cost
|
||||
recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
|
||||
deferred: Final = self.deferred_success_result
|
||||
observed_response: Final = _LITELLM_PARAMS.validate_python(
|
||||
_extract_response_obj_and_hidden_params(deferred[0] if deferred else response, None)[0]
|
||||
)
|
||||
self.model_call_details["_accounting_usage_observed"] = (
|
||||
UsageObservation.KNOWN
|
||||
if (
|
||||
self.model_call_details.get("combined_usage_object")
|
||||
or StandardLoggingPayloadSetup.get_usage_as_dict(observed_response)
|
||||
)
|
||||
else UsageObservation.UNKNOWN
|
||||
)
|
||||
prepared: Final = self.prepare_success_logging(
|
||||
result=deferred[0] if deferred is not None else response,
|
||||
start_time=deferred[1] if deferred is not None else self.start_time,
|
||||
|
|
@ -2540,17 +2619,29 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
},
|
||||
usage,
|
||||
)
|
||||
self.record_partial_usage_for_failure(usage, raw_cost)
|
||||
self.record_partial_usage_for_failure(
|
||||
usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]
|
||||
)
|
||||
return self.prepared_cost
|
||||
|
||||
def record_finalized_cost(self, finalized: FinalizedCost) -> None:
|
||||
self.finalized_cost = finalized
|
||||
self.model_call_details["response_cost"] = finalized.amount
|
||||
|
||||
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
|
||||
def record_usage_capture(self, captured: "CapturedUsage") -> None:
|
||||
self.model_call_details["_accounting_capture"] = captured
|
||||
|
||||
def record_usage_pricing(self, pricing: object) -> None:
|
||||
self.model_call_details["_accounting_pricing_snapshot"] = pricing
|
||||
|
||||
def record_partial_usage_for_failure(
|
||||
self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN
|
||||
) -> None:
|
||||
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
|
||||
self.model_call_details["combined_usage_object"] = usage
|
||||
self.model_call_details["response_cost"] = response_cost
|
||||
if type(self).usage_capture_callback is not None:
|
||||
self.model_call_details["_accounting_usage_observed"] = observation
|
||||
|
||||
def record_failure_after_earlier_rounds(
|
||||
self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
|
||||
|
|
@ -2630,6 +2721,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
own usage while the request's failure log still bills every attempt."""
|
||||
usage: Final = self._model_call_details_view.get("combined_usage_object")
|
||||
response_cost: Final = self._model_call_details_view.get("response_cost")
|
||||
if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
|
||||
self.capture_failed_attempt_usage(usage)
|
||||
self.model_call_details.pop("_accounting_capture", None)
|
||||
self.model_call_details.pop("_accounting_pricing_snapshot", None)
|
||||
self.model_call_details.pop("_accounting_usage_observed", None)
|
||||
self.model_call_details.pop("combined_usage_object", None)
|
||||
self.model_call_details.pop("response_cost", None)
|
||||
if isinstance(usage, Usage):
|
||||
|
|
@ -2639,6 +2735,53 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
|
||||
)
|
||||
|
||||
def capture_failed_attempt_usage(self, usage: Usage) -> None:
|
||||
from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
|
||||
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
|
||||
|
||||
capture: Final = type(self).usage_capture_callback
|
||||
if capture is None:
|
||||
return
|
||||
attempt: Final = self.attempt_scoped_copy()
|
||||
attempt.prepared_cost = None
|
||||
attempt.finalized_cost = None
|
||||
identity: Final = (
|
||||
f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}"
|
||||
)
|
||||
params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {})
|
||||
metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params)
|
||||
observed: Final[object] = self.model_call_details.get("_accounting_usage_observed")
|
||||
cost: Final[object] = self.model_call_details.get("response_cost")
|
||||
attempt.model_call_details = {
|
||||
**attempt.model_call_details,
|
||||
"litellm_call_id": identity,
|
||||
"parent_request_id": self.litellm_call_id,
|
||||
"_accounting_capture": None,
|
||||
"_accounting_prior_captures": (),
|
||||
"_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
|
||||
usage,
|
||||
observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN,
|
||||
float(cost) if isinstance(cost, (int, float)) else None,
|
||||
),
|
||||
"litellm_params": {
|
||||
**params,
|
||||
"metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
|
||||
"litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
|
||||
},
|
||||
}
|
||||
captured: Final = capture(
|
||||
attempt,
|
||||
{"id": identity, "usage": usage.model_dump(), "status": "failed"},
|
||||
self.start_time,
|
||||
datetime.datetime.now(datetime.timezone.utc),
|
||||
False,
|
||||
)
|
||||
if isinstance(captured, CapturedUsage):
|
||||
self.model_call_details["_accounting_prior_captures"] = (
|
||||
*self.model_call_details.get("_accounting_prior_captures", ()),
|
||||
captured,
|
||||
)
|
||||
|
||||
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
|
||||
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
|
||||
self.model_call_details["continued_partial_usage"] = usage
|
||||
|
|
@ -2796,6 +2939,55 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
_update_completion_start_time = update_completion_start_time
|
||||
|
||||
def normalize_endpoint_logging_result(self, result: object) -> object:
|
||||
call_type: Final[object] = self.call_type
|
||||
if call_type == CallTypes.anthropic_messages.value:
|
||||
return self._anthropic_messages_logged_response(result=result)
|
||||
if call_type in (CallTypes.generate_content.value, CallTypes.agenerate_content.value):
|
||||
return self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
|
||||
if call_type in (CallTypes.asend_message.value, CallTypes.send_message.value):
|
||||
return self._handle_a2a_response_logging(result=result)
|
||||
return result
|
||||
|
||||
def usage_capture_response(self, result: object) -> object:
|
||||
if self.stream and not self._is_assembled_stream_success(result):
|
||||
return None
|
||||
copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result
|
||||
normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied))
|
||||
if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list):
|
||||
frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result)
|
||||
measured: Final = any(
|
||||
isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping)
|
||||
for frame in frames
|
||||
if frame.get("type") == "response.done"
|
||||
)
|
||||
if not measured:
|
||||
metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
|
||||
return {
|
||||
"id": self.litellm_call_id,
|
||||
"usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
|
||||
if metered is not None
|
||||
else None,
|
||||
}
|
||||
assembled: Final = self._get_assembled_streaming_response(
|
||||
normalized, self.start_time, self.start_time, True, []
|
||||
)
|
||||
return normalized if assembled is None else assembled
|
||||
|
||||
def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
|
||||
from litellm.utils import count_characters
|
||||
|
||||
return TypeAdapter(dict[str, JsonValue]).validate_python(
|
||||
{
|
||||
"custom_pricing": self.custom_pricing_for(result),
|
||||
"custom_cost_per_token": self.custom_cost_per_token,
|
||||
"custom_cost_per_second": self.custom_cost_per_second,
|
||||
"prompt_characters": count_characters(self._prompt_for_cost_calculation())
|
||||
if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
|
||||
else None,
|
||||
}
|
||||
)
|
||||
|
||||
def normalize_logging_result(self, result: object) -> object:
|
||||
"""
|
||||
Some endpoints return a different type of result than what is expected by the logging system.
|
||||
|
|
@ -2806,8 +2998,20 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
combined_usage_object: Final = (
|
||||
RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=result)
|
||||
)
|
||||
metered: Final[object] = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
|
||||
captured_usage: Final = (
|
||||
combined_usage_object.model_copy(
|
||||
update={
|
||||
"metered_usage": TypeAdapter(dict[str, JsonValue]).validate_python(
|
||||
TypeAdapter(object).dump_python(metered, mode="json")
|
||||
)
|
||||
}
|
||||
)
|
||||
if metered is not None
|
||||
else combined_usage_object
|
||||
)
|
||||
logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object(
|
||||
usage=combined_usage_object,
|
||||
usage=captured_usage,
|
||||
results=result,
|
||||
)
|
||||
|
||||
|
|
@ -3068,7 +3272,14 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
start_time = self.start_time
|
||||
if end_time is None:
|
||||
end_time = datetime.datetime.now()
|
||||
if self.completion_start_time is None:
|
||||
# A request re-dispatched to a fallback reuses this logging object but runs in
|
||||
# its own window, so a first-token time from the previous attempt cannot belong
|
||||
# to it. Keeping it makes the attempt report a negative TTFT.
|
||||
first_token_seconds: Final = _timestamp_seconds(self.completion_start_time)
|
||||
attempt_start_seconds: Final = _timestamp_seconds(start_time)
|
||||
if first_token_seconds is None or (
|
||||
attempt_start_seconds is not None and first_token_seconds < attempt_start_seconds
|
||||
):
|
||||
self.completion_start_time = end_time
|
||||
self.model_call_details["completion_start_time"] = self.completion_start_time
|
||||
|
||||
|
|
@ -3080,15 +3291,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
else cache_hit
|
||||
)
|
||||
|
||||
if self.call_type == CallTypes.anthropic_messages.value:
|
||||
result = self._anthropic_messages_logged_response(result=result)
|
||||
elif (
|
||||
self.call_type == CallTypes.generate_content.value
|
||||
or self.call_type == CallTypes.agenerate_content.value
|
||||
):
|
||||
result = self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
|
||||
elif self.call_type == CallTypes.asend_message.value or self.call_type == CallTypes.send_message.value:
|
||||
result = self._handle_a2a_response_logging(result=result)
|
||||
result = self.normalize_endpoint_logging_result(result)
|
||||
|
||||
logging_result: Final = self.normalize_logging_result(result=result)
|
||||
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
|
||||
|
|
@ -3441,6 +3644,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._with_client_facing_stream_model(complete_streaming_response)
|
||||
)
|
||||
self.model_call_details["complete_streaming_response"] = logged_streaming_response
|
||||
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
|
||||
## STANDARDIZED LOGGING PAYLOAD
|
||||
if not for_async_request:
|
||||
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
|
||||
|
|
@ -3964,6 +4168,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
cost_for_built_in_tools_cost_usd_dollar=0.0,
|
||||
)
|
||||
|
||||
self.capture_usage_before_logging(result, start_time, end_time)
|
||||
|
||||
self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
|
||||
StandardLoggingPayloadSetup.append_system_prompt_messages(
|
||||
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
|
||||
|
|
@ -4990,16 +5196,34 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
def billable_result(self, result: object) -> object:
|
||||
"""The response the cost calculator prices: a /v1/messages body served through the chat
|
||||
bridge is replaced by the ModelResponse its spend row is built from, anything else is
|
||||
bridge is replaced by the ModelResponse its spend row is built from, a native one is only
|
||||
converted when its provider reports the cost the bill is read from, anything else is
|
||||
priced as is."""
|
||||
served: Final[object] = result
|
||||
call_type: Final[object] = getattr(self, "call_type", None)
|
||||
if not isinstance(result, dict):
|
||||
return served
|
||||
if self.bridged_model_response is None and call_type != CallTypes.anthropic_messages.value:
|
||||
if self.bridged_model_response is not None:
|
||||
return self._anthropic_messages_logged_response(result=served)
|
||||
if call_type != CallTypes.anthropic_messages.value:
|
||||
return served
|
||||
if not reports_native_messages_cost(self.model_call_details.get("custom_llm_provider")):
|
||||
return self._served_with_requested_speed(result=served)
|
||||
return self._anthropic_messages_logged_response(result=served)
|
||||
|
||||
def _served_with_requested_speed(self, result: object) -> object:
|
||||
"""A native /v1/messages body priced as served bills the speed the client asked for when its
|
||||
usage block reports none, the same speed its spend row is priced at."""
|
||||
body: Final = _SERVED_ANTHROPIC_BODY.validate_python(result)
|
||||
usage: Final = body.get("usage")
|
||||
requested: Final = _LITELLM_PARAMS.validate_python(getattr(self, "optional_params", None) or {}).get("speed")
|
||||
if not isinstance(requested, str) or not isinstance(usage, dict):
|
||||
return result
|
||||
reported: Final = _SERVED_ANTHROPIC_BODY.validate_python(usage)
|
||||
if isinstance(reported.get("speed"), str):
|
||||
return result
|
||||
return {**body, "usage": {**reported, "speed": requested}}
|
||||
|
||||
def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
|
||||
"""
|
||||
The ModelResponse a /v1/messages spend_logs row is built from.
|
||||
|
|
@ -6808,45 +7032,19 @@ class StandardLoggingPayloadSetup:
|
|||
|
||||
@staticmethod
|
||||
def get_usage_from_response_obj(response_obj: dict | None, combined_usage_object: Usage | None = None) -> Usage:
|
||||
## BASE CASE ##
|
||||
if combined_usage_object is not None:
|
||||
return combined_usage_object
|
||||
if response_obj is None:
|
||||
return Usage(
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
total_tokens=0,
|
||||
)
|
||||
|
||||
usage: Final = response_obj.get("usage", None)
|
||||
if usage is None:
|
||||
rerank_usage: Final = get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj))
|
||||
if rerank_usage is not None:
|
||||
return Usage(**rerank_usage)
|
||||
if usage is None or (not isinstance(usage, dict) and not isinstance(usage, Usage)):
|
||||
return Usage(
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
total_tokens=0,
|
||||
)
|
||||
elif isinstance(usage, Usage):
|
||||
return usage
|
||||
elif isinstance(usage, ResponseAPIUsage):
|
||||
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
|
||||
elif isinstance(usage, dict):
|
||||
if ResponseAPILoggingUtils.is_response_api_usage(usage):
|
||||
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
|
||||
if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
|
||||
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
|
||||
return Usage(**usage)
|
||||
|
||||
raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
|
||||
reported: Final = response_obj.get("usage") if response_obj else None
|
||||
if isinstance(reported, Usage):
|
||||
return reported
|
||||
normalized: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj)
|
||||
return Usage(**normalized) if normalized else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
|
||||
|
||||
@staticmethod
|
||||
def get_usage_as_dict(
|
||||
response_obj: dict | None,
|
||||
response_obj: dict[str, Any] | None,
|
||||
combined_usage_object: Usage | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Like get_usage_from_response_obj but returns a plain dict, skipping
|
||||
the Pydantic Usage construction on the hot path.
|
||||
|
|
@ -6858,7 +7056,21 @@ class StandardLoggingPayloadSetup:
|
|||
return _empty
|
||||
_raw: Final = response_obj.get("usage", None)
|
||||
if _raw is None:
|
||||
return dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
|
||||
google_usage: Final[object] = response_obj.get("usageMetadata")
|
||||
if isinstance(google_usage, dict) and _GOOGLE_USAGE_FIELDS.intersection(google_usage):
|
||||
return litellm.VertexGeminiConfig.calculate_usage(
|
||||
cast( # cast-ok: retain converter-supported modalities and token_count aliases
|
||||
GenerateContentResponseBody, response_obj
|
||||
)
|
||||
).model_dump()
|
||||
reported: Final = response_obj.get("usage_info")
|
||||
if isinstance(reported, BaseModel):
|
||||
return reported.model_dump(exclude_none=True)
|
||||
return (
|
||||
{key: value for key, value in reported.items() if value is not None}
|
||||
if isinstance(reported, dict)
|
||||
else dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
|
||||
)
|
||||
if isinstance(_raw, ResponseAPIUsage):
|
||||
return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
|
||||
if isinstance(_raw, dict):
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from openai.types.responses.response_create_params import (
|
|||
ResponseCreateParamsNonStreaming,
|
||||
ResponseCreateParamsStreaming,
|
||||
)
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.llms.anthropic import AnthropicMessagesRequest
|
||||
|
|
@ -41,6 +42,16 @@ def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> obj
|
|||
return None
|
||||
|
||||
|
||||
_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None)
|
||||
|
||||
|
||||
def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool:
|
||||
try:
|
||||
return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False
|
||||
except ValidationError:
|
||||
return True
|
||||
|
||||
|
||||
class ModelParamHelper:
|
||||
# Cached at class level — deterministic set built from static OpenAI type annotations
|
||||
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
|
|||
return any(message.get(key, None) is not None for key in message if key not in ignore_keys)
|
||||
|
||||
|
||||
_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
|
||||
_IMAGE_CONTENT_PART_TYPES: Final = ("image_url", "input_image", "image", "computer_screenshot")
|
||||
_IMAGE_SCAN_MAX_DEPTH: Final = 4
|
||||
|
||||
|
||||
|
|
@ -557,15 +557,24 @@ def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObjec
|
|||
return {"type": "image_url", "image_url": {"url": url}} if url else None
|
||||
|
||||
|
||||
def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
|
||||
"""Whether any message carries an image content part, across the dialects that reach
|
||||
pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,
|
||||
and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks."""
|
||||
return any(
|
||||
isinstance(content, list) and _content_parts_contain_image(content)
|
||||
for message in messages
|
||||
for content in (message.get("content"),)
|
||||
)
|
||||
def _message_contains_image_content(message: object) -> bool:
|
||||
match message:
|
||||
case {"content": list()}:
|
||||
return _content_parts_contain_image(content_parts(_CONTENT_PART.validate_python(message)))
|
||||
case {"type": "function_call_output" | "custom_tool_call_output" | "tool_result", "output": list()}:
|
||||
output: Final = _CONTENT_PART.validate_python(message)["output"]
|
||||
return _content_parts_contain_image(_CONTENT_PARTS.validate_python(output))
|
||||
case {"type": "computer_call_output", "output": dict()}:
|
||||
return _content_parts_contain_image((_CONTENT_PART.validate_python(message)["output"],))
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def request_contains_image_content(messages: object) -> bool:
|
||||
"""Inspect native message content without translating or validating the request."""
|
||||
if not isinstance(messages, (list, tuple)):
|
||||
return False
|
||||
return any(_message_contains_image_content(message) for message in _CONTENT_PARTS.validate_python(messages))
|
||||
|
||||
|
||||
def audio_or_image_in_message_content(message: AllMessageValues) -> bool:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
|
|||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
|
||||
anthropic_system_messages,
|
||||
find_anthropic_server_tool_result,
|
||||
is_anthropic_server_tool_use,
|
||||
is_anthropic_tool_search_use,
|
||||
message_field,
|
||||
parts_of,
|
||||
|
|
@ -1252,6 +1253,22 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str
|
|||
_encode_tool_call_id_with_signature = encode_tool_call_id_with_signature
|
||||
|
||||
|
||||
def _tool_call_id_prefix(tool_call_id: object) -> str | None:
|
||||
return tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] if isinstance(tool_call_id, str) else None
|
||||
|
||||
|
||||
def _called_tool_name(tool_call_id: str, tool_calls: Sequence[Mapping[str, object]]) -> str | None:
|
||||
exact_matches: Final = tuple(call for call in tool_calls if call.get("id") == tool_call_id)
|
||||
prefix: Final = _tool_call_id_prefix(tool_call_id)
|
||||
prefix_matches: Final = tuple(
|
||||
call for call in tool_calls if prefix and _tool_call_id_prefix(call.get("id")) == prefix
|
||||
)
|
||||
matches: Final = exact_matches or (prefix_matches if len(prefix_matches) == 1 else ())
|
||||
function: Final = matches[0].get("function") if matches else None
|
||||
name: Final = function.get("name") if isinstance(function, Mapping) else None
|
||||
return name if isinstance(name, str) else None
|
||||
|
||||
|
||||
def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
|
||||
"""Extract thought signature from tool call's provider_specific_fields.
|
||||
|
||||
|
|
@ -1691,16 +1708,14 @@ def convert_to_gemini_tool_call_result(
|
|||
)
|
||||
) is not None:
|
||||
media_parts.append(media_part)
|
||||
name: str | None = message.get("name", "")
|
||||
|
||||
# Recover name from last message with tool calls
|
||||
if last_message_with_tool_calls:
|
||||
tools: Final = last_message_with_tool_calls.get("tool_calls", [])
|
||||
msg_tool_call_id: Final = message.get("tool_call_id", None)
|
||||
for tool in tools:
|
||||
prev_tool_call_id = tool.get("id", None)
|
||||
if msg_tool_call_id and prev_tool_call_id and msg_tool_call_id == prev_tool_call_id:
|
||||
name = tool.get("function", {}).get("name", "")
|
||||
msg_tool_call_id: Final = message.get("tool_call_id", None)
|
||||
called_tool_name: Final = (
|
||||
_called_tool_name(msg_tool_call_id, last_message_with_tool_calls.get("tool_calls") or ())
|
||||
if last_message_with_tool_calls and isinstance(msg_tool_call_id, str) and msg_tool_call_id
|
||||
else None
|
||||
)
|
||||
name: Final[str | None] = called_tool_name if called_tool_name is not None else message.get("name", "")
|
||||
|
||||
# Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
|
||||
gemini_call_id: str | None = None
|
||||
|
|
@ -2059,6 +2074,54 @@ def _placed_and_paired(
|
|||
_AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
|
||||
|
||||
|
||||
def _is_server_tool_result(block: Mapping[str, object]) -> bool:
|
||||
block_type: Final = block.get("type")
|
||||
return isinstance(block_type, str) and block_type.endswith("_tool_result")
|
||||
|
||||
|
||||
def _split_unrun_server_tool_uses(
|
||||
groups: Sequence[Sequence[_AnthropicToolInvokeItem]],
|
||||
) -> tuple[tuple[Sequence[_AnthropicToolInvokeItem], ...], tuple[_AnthropicToolInvokeItem, ...]]:
|
||||
completed: Final = tuple(group for group in groups if any(_is_server_tool_result(block) for block in group))
|
||||
unrun: Final = tuple(group[0] for group in groups if not any(_is_server_tool_result(block) for block in group))
|
||||
return completed, unrun
|
||||
|
||||
|
||||
def _block_id(block: Mapping[str, object]) -> object:
|
||||
return block.get("id")
|
||||
|
||||
|
||||
def _unrun_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozenset[object]:
|
||||
result_ids: Final = frozenset(block.get("tool_use_id") for block in blocks if _is_server_tool_result(block))
|
||||
return frozenset(
|
||||
_block_id(block)
|
||||
for block in blocks
|
||||
if block.get("type") == "server_tool_use" and _block_id(block) not in result_ids
|
||||
)
|
||||
|
||||
|
||||
def _last_block_id_among(blocks: Sequence[Mapping[str, object]], ids: frozenset[object]) -> object:
|
||||
return next(block_id for block in reversed(blocks) if (block_id := _block_id(block)) in ids)
|
||||
|
||||
|
||||
def _paused_turn_replay(
|
||||
tool_invokes: Sequence[_AnthropicToolInvokeItem],
|
||||
thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
|
||||
*,
|
||||
is_final_message: bool,
|
||||
) -> tuple[
|
||||
tuple[_AnthropicToolInvokeItem, ...],
|
||||
tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...],
|
||||
]:
|
||||
unrun_ids: Final = _unrun_server_tool_use_ids(tool_invokes)
|
||||
if not unrun_ids:
|
||||
return tuple(tool_invokes), tuple(thinking_blocks)
|
||||
dropped_ids: Final = (
|
||||
unrun_ids - frozenset((_last_block_id_among(tool_invokes, unrun_ids),)) if is_final_message else unrun_ids
|
||||
)
|
||||
return tuple(block for block in tool_invokes if _block_id(block) not in dropped_ids), ()
|
||||
|
||||
|
||||
def convert_to_anthropic_tool_invoke(
|
||||
tool_calls: list[ChatCompletionAssistantToolCall],
|
||||
web_search_results: Sequence[object] | None = None,
|
||||
|
|
@ -2141,6 +2204,15 @@ def convert_to_anthropic_tool_invoke(
|
|||
anthropic_tool_invoke.append(server_tool_result)
|
||||
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
|
||||
continue
|
||||
elif is_anthropic_server_tool_use(tool_id) and tool_id not in answered_tool_ids:
|
||||
anthropic_tool_invoke.append(
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": tool_id,
|
||||
"name": tool_name,
|
||||
"input": tool_input,
|
||||
}
|
||||
)
|
||||
else:
|
||||
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
|
||||
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
|
||||
|
|
@ -3109,6 +3181,30 @@ def anthropic_messages_pt(
|
|||
_has_server_tool_calls = True
|
||||
break
|
||||
|
||||
# Get web_search_results and tool_results from provider_specific_fields
|
||||
# for server_tool_use reconstruction.
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
|
||||
_provider_specific_fields: dict[str, Any] = (
|
||||
cast(dict[str, Any], _provider_specific_fields_raw)
|
||||
if isinstance(_provider_specific_fields_raw, dict)
|
||||
else {}
|
||||
)
|
||||
tool_invoke_results = (
|
||||
convert_to_anthropic_tool_invoke(
|
||||
assistant_tool_calls,
|
||||
web_search_results=_provider_specific_fields.get("web_search_results"),
|
||||
tool_results=_provider_specific_fields.get("tool_results"),
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
)
|
||||
if assistant_tool_calls is not None
|
||||
else ()
|
||||
)
|
||||
turn_paused = bool(_unrun_server_tool_use_ids(tool_invoke_results))
|
||||
replayed_tool_invokes, replayed_thinking_blocks = _paused_turn_replay(
|
||||
tool_invoke_results, thinking_blocks or (), is_final_message=msg_i == len(messages) - 1
|
||||
)
|
||||
|
||||
if (
|
||||
thinking_blocks is not None
|
||||
and _has_server_tool_calls
|
||||
|
|
@ -3121,26 +3217,12 @@ def anthropic_messages_pt(
|
|||
# We must preserve this interleaved order because Anthropic
|
||||
# verifies thinking block signatures based on position.
|
||||
|
||||
# Build the tool call groups (server_tool_use + its result)
|
||||
_provider_specific_fields_raw_tc = assistant_content_block.get("provider_specific_fields")
|
||||
_provider_specific_fields_tc: dict[str, Any] = {}
|
||||
if isinstance(_provider_specific_fields_raw_tc, dict):
|
||||
_provider_specific_fields_tc = cast(dict[str, Any], _provider_specific_fields_raw_tc)
|
||||
_web_search_results_tc = _provider_specific_fields_tc.get("web_search_results")
|
||||
_tool_results_tc = _provider_specific_fields_tc.get("tool_results")
|
||||
tool_invoke_results = convert_to_anthropic_tool_invoke(
|
||||
assistant_tool_calls,
|
||||
web_search_results=_web_search_results_tc,
|
||||
tool_results=_tool_results_tc,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
)
|
||||
|
||||
# Group tool invoke results into (server_tool_use, result) pairs
|
||||
# and separate regular tool_use blocks
|
||||
server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
|
||||
regular_tool_uses: list[_AnthropicToolInvokeItem] = []
|
||||
_current_group: list[_AnthropicToolInvokeItem] = []
|
||||
for item in tool_invoke_results:
|
||||
for item in replayed_tool_invokes:
|
||||
item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
|
||||
if item_type == "server_tool_use":
|
||||
if _current_group:
|
||||
|
|
@ -3154,6 +3236,9 @@ def anthropic_messages_pt(
|
|||
_current_group.append(item)
|
||||
if _current_group:
|
||||
server_tool_groups.append(_current_group)
|
||||
completed_server_tool_groups, pending_server_tool_uses = _split_unrun_server_tool_uses(
|
||||
server_tool_groups
|
||||
)
|
||||
|
||||
# Build the text block if content is a non-empty string
|
||||
text_element = None
|
||||
|
|
@ -3177,15 +3262,15 @@ def anthropic_messages_pt(
|
|||
# Any remaining groups (after all thinking blocks) go after.
|
||||
tb_idx = 0
|
||||
grp_idx = 0
|
||||
num_tb = len(thinking_blocks) if thinking_blocks else 0
|
||||
num_grp = len(server_tool_groups)
|
||||
num_tb = len(replayed_thinking_blocks)
|
||||
num_grp = len(completed_server_tool_groups)
|
||||
|
||||
while tb_idx < num_tb or grp_idx < num_grp:
|
||||
if tb_idx < num_tb and grp_idx < num_grp:
|
||||
# Emit thinking block then its tool group
|
||||
assistant_content.append(thinking_blocks[tb_idx])
|
||||
assistant_content.append(replayed_thinking_blocks[tb_idx])
|
||||
tb_idx += 1
|
||||
for block in server_tool_groups[grp_idx]:
|
||||
for block in completed_server_tool_groups[grp_idx]:
|
||||
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
|
||||
if item_id and item_id in unique_tool_ids:
|
||||
continue
|
||||
|
|
@ -3195,11 +3280,11 @@ def anthropic_messages_pt(
|
|||
grp_idx += 1
|
||||
elif tb_idx < num_tb:
|
||||
# More thinking blocks than tool groups - emit before text
|
||||
assistant_content.append(thinking_blocks[tb_idx])
|
||||
assistant_content.append(replayed_thinking_blocks[tb_idx])
|
||||
tb_idx += 1
|
||||
else:
|
||||
# More tool groups than thinking blocks - emit remaining
|
||||
for block in server_tool_groups[grp_idx]:
|
||||
for block in completed_server_tool_groups[grp_idx]:
|
||||
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
|
||||
if item_id and item_id in unique_tool_ids:
|
||||
continue
|
||||
|
|
@ -3212,8 +3297,8 @@ def anthropic_messages_pt(
|
|||
if text_element is not None:
|
||||
assistant_content.append(text_element)
|
||||
|
||||
# Add regular (non-server) tool calls at the end
|
||||
for item in regular_tool_uses:
|
||||
# Add regular tool calls, then a paused turn's unrun server tool call, at the end
|
||||
for item in (*regular_tool_uses, *pending_server_tool_uses):
|
||||
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
|
||||
if item_id and item_id in unique_tool_ids:
|
||||
continue
|
||||
|
|
@ -3233,18 +3318,18 @@ def anthropic_messages_pt(
|
|||
)
|
||||
_content_list = assistant_content_block.get("content") if _content_is_list else None
|
||||
|
||||
if (
|
||||
thinking_blocks is not None and not _content_is_list
|
||||
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
|
||||
assistant_content.extend(thinking_blocks)
|
||||
if not _content_is_list: # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
|
||||
assistant_content.extend(replayed_thinking_blocks)
|
||||
if _content_is_list and _content_list is not None:
|
||||
for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()):
|
||||
for m in _with_missing_thinking_blocks(_content_list, replayed_thinking_blocks):
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
# handle thinking blocks
|
||||
if (thinking_block := _as_thinking_block(m)) is not None:
|
||||
assistant_content.extend(
|
||||
_replayable_thinking_blocks([thinking_block], drop_params=drop_params)
|
||||
()
|
||||
if turn_paused
|
||||
else _replayable_thinking_blocks([thinking_block], drop_params=drop_params)
|
||||
)
|
||||
continue
|
||||
text_block = cast(str, m.get("text", ""))
|
||||
|
|
@ -3284,25 +3369,9 @@ def anthropic_messages_pt(
|
|||
assistant_content.append(_anthropic_text_content_element)
|
||||
|
||||
if assistant_tool_calls is not None: # support assistant tool invoke conversion
|
||||
# Get web_search_results and tool_results from provider_specific_fields
|
||||
# for server_tool_use reconstruction.
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/17737
|
||||
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
|
||||
_provider_specific_fields: dict[str, Any] = {}
|
||||
if isinstance(_provider_specific_fields_raw, dict):
|
||||
_provider_specific_fields = cast(dict[str, Any], _provider_specific_fields_raw)
|
||||
_web_search_results = _provider_specific_fields.get("web_search_results")
|
||||
_tool_results = _provider_specific_fields.get("tool_results")
|
||||
tool_invoke_results = convert_to_anthropic_tool_invoke(
|
||||
assistant_tool_calls,
|
||||
web_search_results=_web_search_results,
|
||||
tool_results=_tool_results,
|
||||
answered_tool_ids=client_answered_tool_ids,
|
||||
)
|
||||
|
||||
# Prevent "tool_use ids must be unique" errors by filtering duplicates
|
||||
# This can happen when merging history that already contains the tool calls
|
||||
for item in tool_invoke_results:
|
||||
for item in replayed_tool_invokes:
|
||||
# tool_use items are typically dicts, but handle objects just in case
|
||||
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
|
||||
|
||||
|
|
|
|||
|
|
@ -286,6 +286,10 @@ def _separate_thinking_blocks_render(message: object) -> bool:
|
|||
return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks")))
|
||||
|
||||
|
||||
def is_anthropic_server_tool_use(tool_id: str) -> bool:
|
||||
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
|
||||
|
||||
|
||||
def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
|
||||
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES
|
||||
|
||||
|
|
|
|||
|
|
@ -631,6 +631,8 @@ class RealTimeStreaming:
|
|||
if self.session_tools or self.tool_calls:
|
||||
self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
|
||||
self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
|
||||
if LiteLLMLogging.usage_capture_callback is not None:
|
||||
self.logging_obj.capture_usage_before_logging(self.messages)
|
||||
# Route through the bounded logging worker (per-coroutine timeout +
|
||||
# concurrency cap) instead of a bare create_task, so a slow callback
|
||||
# can't leave suspended tasks pinning each call's response in memory.
|
||||
|
|
|
|||
|
|
@ -1509,11 +1509,20 @@ def _delta_reasoning_items(delta: object) -> Sequence[object]:
|
|||
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
|
||||
|
||||
|
||||
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
|
||||
def _chunk_first_delta(chunk: object) -> object:
|
||||
choices: Final = _field_of(chunk, "choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
return ()
|
||||
return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk
|
||||
return None
|
||||
return _field_of(cast(Sequence[object], choices)[0], "delta") # cast-ok: untyped chunk
|
||||
|
||||
|
||||
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
|
||||
return _delta_reasoning_items(_chunk_first_delta(chunk))
|
||||
|
||||
|
||||
def _chunk_refusal(chunk: object) -> str:
|
||||
refusal: Final = _field_of(_chunk_first_delta(chunk), "refusal")
|
||||
return refusal if isinstance(refusal, str) else ""
|
||||
|
||||
|
||||
_NON_TEXT_DELTA_FIELDS: Final = (
|
||||
|
|
@ -1526,6 +1535,7 @@ _NON_TEXT_DELTA_FIELDS: Final = (
|
|||
"audio",
|
||||
"images",
|
||||
"provider_specific_fields",
|
||||
"refusal",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1674,6 +1684,9 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It
|
|||
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
|
||||
|
||||
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
|
||||
combined_refusal: Final = "".join(_chunk_refusal(chunk) for chunk in typed_chunks)
|
||||
if combined_refusal:
|
||||
yield "refusal", combined_refusal
|
||||
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
|
||||
if reasoning_items:
|
||||
yield "reasoning_items", list(reasoning_items)
|
||||
|
|
|
|||
|
|
@ -349,11 +349,14 @@ class CustomStreamWrapper:
|
|||
make_call: Callable | None = None,
|
||||
_response_headers: dict | httpx.Headers | None = None,
|
||||
count_prompt_tokens: Callable[[], int] | None = None,
|
||||
is_mock: bool = False,
|
||||
):
|
||||
self.model = model
|
||||
self.make_call = make_call
|
||||
self.count_prompt_tokens = count_prompt_tokens
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
self.is_mock = is_mock
|
||||
self.uses_openai_chat_stream = False
|
||||
self.logging_obj: LiteLLMLoggingObject = logging_obj
|
||||
self.completion_stream = completion_stream
|
||||
self.sent_first_chunk = False
|
||||
|
|
@ -376,6 +379,7 @@ class CustomStreamWrapper:
|
|||
self._upstream_provider: str | None = None
|
||||
self.received_finish_reason: str | None = None
|
||||
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
|
||||
self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason
|
||||
self.special_tokens = [
|
||||
"<|assistant|>",
|
||||
"<|system|>",
|
||||
|
|
@ -1302,8 +1306,8 @@ class CustomStreamWrapper:
|
|||
if _is_delta_empty:
|
||||
model_response.choices[0].delta = Delta(content=self._close_open_think_block())
|
||||
# get any function call arguments
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
finish_reason=self.received_finish_reason
|
||||
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
|
||||
map_finish_reason(finish_reason=self.received_finish_reason)
|
||||
) # ensure consistent output to openai
|
||||
|
||||
self.sent_last_chunk = True
|
||||
|
|
@ -1597,6 +1601,7 @@ class CustomStreamWrapper:
|
|||
if response_obj["is_finished"]:
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
else: # openai / azure chat model
|
||||
self.uses_openai_chat_stream = True
|
||||
if isinstance(chunk, ChatCompletionChunk):
|
||||
self._openai_handler_receives_chat_chunks = True
|
||||
if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
|
||||
|
|
@ -1617,6 +1622,7 @@ class CustomStreamWrapper:
|
|||
return _ProviderChunkEarlyReturn(None)
|
||||
completion_obj["content"] = response_obj["text"]
|
||||
self.intermittent_finish_reason = response_obj.get("finish_reason", None)
|
||||
self.received_any_finish_reason |= response_obj["finish_reason"] is not None
|
||||
if response_obj["is_finished"]:
|
||||
if response_obj["finish_reason"] == "error":
|
||||
raise Exception(
|
||||
|
|
@ -1737,6 +1743,8 @@ class CustomStreamWrapper:
|
|||
# if function returned but type set to None - mistral's api returns type: None
|
||||
tool["type"] = "function"
|
||||
model_response.choices[0].delta = Delta(**_json_delta)
|
||||
if model_response.choices[0].delta.tool_calls:
|
||||
self.tool_call = True
|
||||
except Exception as e:
|
||||
verbose_logger.exception(
|
||||
"litellm.CustomStreamWrapper.chunk_creator(): Exception occured - %s", e
|
||||
|
|
@ -1966,6 +1974,11 @@ class CustomStreamWrapper:
|
|||
if self.logging_obj.is_sync_litellm_request(litellm_params):
|
||||
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
|
||||
|
||||
def _normalize_tool_call_finish_reason(self, finish_reason: str) -> str:
|
||||
if finish_reason == "stop" and self.tool_call:
|
||||
return "tool_calls"
|
||||
return finish_reason
|
||||
|
||||
def finish_reason_handler(self):
|
||||
model_response: Final = self.model_response_creator(
|
||||
hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
|
||||
|
|
@ -1981,21 +1994,30 @@ class CustomStreamWrapper:
|
|||
if served_tier is not None:
|
||||
setattr(model_response, "service_tier", served_tier) # noqa: B010 # pydantic extra, not a declared field
|
||||
_finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
|
||||
if _finish_reason is not None:
|
||||
model_response.choices[0].finish_reason = _finish_reason
|
||||
else:
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
|
||||
## if tool use
|
||||
if (
|
||||
model_response.choices[0].finish_reason == "stop" and self.tool_call
|
||||
): # don't overwrite for other - potential error finish reasons
|
||||
model_response.choices[0].finish_reason = "tool_calls"
|
||||
model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
|
||||
_finish_reason if _finish_reason is not None else "stop"
|
||||
)
|
||||
close_tag: Final = self._close_open_think_block()
|
||||
if close_tag is not None:
|
||||
model_response.choices[0].delta.content = close_tag
|
||||
return model_response
|
||||
|
||||
def _should_fail_on_clean_eof(self) -> bool:
|
||||
return (
|
||||
not self.is_mock
|
||||
and self.uses_openai_chat_stream
|
||||
and self.received_finish_reason is None
|
||||
and self.intermittent_finish_reason is None
|
||||
and not self.received_any_finish_reason
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stream_eof_error() -> httpx.RemoteProtocolError:
|
||||
return httpx.RemoteProtocolError(
|
||||
"Stream ended before a finish_reason was received",
|
||||
request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs"),
|
||||
)
|
||||
|
||||
def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
|
||||
if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
|
||||
return processed_chunk
|
||||
|
|
@ -2181,6 +2203,13 @@ class CustomStreamWrapper:
|
|||
return response
|
||||
|
||||
except StopIteration:
|
||||
if self._should_fail_on_clean_eof():
|
||||
error: Final = self._stream_eof_error()
|
||||
eof_traceback: Final = traceback.format_exc()
|
||||
self._record_partial_usage_for_failure()
|
||||
threading.Thread(target=self.logging_obj.failure_handler, args=(error, eof_traceback)).start()
|
||||
self.close()
|
||||
self._handle_stream_fallback_error(error)
|
||||
if self.sent_last_chunk is True:
|
||||
complete_streaming_response: Final = self._build_complete_streaming_response()
|
||||
response = self._new_usage_chunk()
|
||||
|
|
@ -2384,6 +2413,9 @@ class CustomStreamWrapper:
|
|||
continue
|
||||
return usage_free_chunk
|
||||
except (StopAsyncIteration, StopIteration):
|
||||
if self._should_fail_on_clean_eof():
|
||||
await self.aclose()
|
||||
self._log_stream_failure_and_raise(self._stream_eof_error())
|
||||
return await self._finalize_completed_stream(cache_hit=cache_hit)
|
||||
except httpx.TimeoutException as e: # if httpx read timeout error occues
|
||||
traceback_exception = traceback.format_exc()
|
||||
|
|
|
|||
|
|
@ -160,8 +160,11 @@ async def resolve_a2a_hop_auth_header(
|
|||
litellm_params: Mapping[str, object],
|
||||
custom_llm_provider: object,
|
||||
resolve_entra_header: AgentAuthHeaderResolver = resolve_azure_ai_agent_auth_header,
|
||||
agent_extra_headers: Mapping[str, str] | None = None,
|
||||
) -> Mapping[str, str] | None:
|
||||
"""Entra credentials authenticate the A2A hop only; a completion-bridge agent hands them to the model provider it bridges to."""
|
||||
if not a2a_hop_uses_entra(litellm_params, custom_llm_provider):
|
||||
if a2a_hop_uses_entra(litellm_params, custom_llm_provider):
|
||||
return await resolve_entra_header(litellm_params)
|
||||
if custom_llm_provider or any(name.lower() == "authorization" for name in (agent_extra_headers or {})):
|
||||
return None
|
||||
return await resolve_entra_header(litellm_params)
|
||||
api_key: Final = litellm_params.get("api_key")
|
||||
return {"Authorization": f"Bearer {api_key}"} if isinstance(api_key, str) and api_key else None
|
||||
|
|
|
|||
|
|
@ -70,7 +70,12 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
from ...base import BaseLLM
|
||||
from ..common_utils import AnthropicError, process_anthropic_headers, refusal_provider_specific_fields
|
||||
from ..common_utils import (
|
||||
AnthropicError,
|
||||
process_anthropic_headers,
|
||||
refusal_params,
|
||||
stop_reason_provider_specific_fields,
|
||||
)
|
||||
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -926,6 +931,17 @@ class ModelResponseIterator:
|
|||
def _web_search_call_snapshot(self) -> dict[str, object]:
|
||||
return dict(self._web_search_calls)
|
||||
|
||||
def _refresh_pending_web_search_call(self, tool_use_id: str) -> bool:
|
||||
if tool_use_id not in self._web_search_calls:
|
||||
return False
|
||||
self._web_search_calls[tool_use_id] = build_web_search_call(
|
||||
tool_use_id,
|
||||
self._server_tool_inputs.get(tool_use_id, {}),
|
||||
{"content": []},
|
||||
status="in_progress",
|
||||
)
|
||||
return True
|
||||
|
||||
def _complete_web_search_call(self, result: dict[str, object]) -> None:
|
||||
tool_use_id: Final = result.get("tool_use_id")
|
||||
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
|
||||
|
|
@ -1128,6 +1144,8 @@ class ModelResponseIterator:
|
|||
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
if self._refresh_pending_web_search_call(self._current_server_tool_id):
|
||||
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
|
||||
self._current_server_tool_id = None
|
||||
if self.is_response_format_tool:
|
||||
text = self._flush_response_format_buffer()
|
||||
|
|
@ -1140,8 +1158,8 @@ class ModelResponseIterator:
|
|||
# These are automatically handled by Anthropic API, we just pass them through
|
||||
pass
|
||||
elif type_chunk == "message_delta":
|
||||
finish_reason, usage, container, refusal_fields = self._handle_message_delta(chunk)
|
||||
provider_specific_fields.update(refusal_fields)
|
||||
finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk)
|
||||
provider_specific_fields.update(stop_reason_fields)
|
||||
text = self._release_answer_close() + self._drain_split_answer_blocks()
|
||||
if container:
|
||||
provider_specific_fields["container"] = container
|
||||
|
|
@ -1190,6 +1208,7 @@ class ModelResponseIterator:
|
|||
)
|
||||
|
||||
text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
|
||||
refusal_fields: Final = refusal_params(provider_specific_fields)
|
||||
|
||||
returned_chunk: Final = ModelResponseStream(
|
||||
choices=[
|
||||
|
|
@ -1201,6 +1220,7 @@ class ModelResponseIterator:
|
|||
provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
|
||||
thinking_blocks=(thinking_blocks if thinking_blocks else None),
|
||||
reasoning_content=reasoning_content,
|
||||
**refusal_fields,
|
||||
),
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
|
|
@ -1371,7 +1391,7 @@ class ModelResponseIterator:
|
|||
self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) if "usage" in message_delta else None
|
||||
)
|
||||
container: Final = message_delta["delta"].get("container")
|
||||
return finish_reason, usage, container, refusal_provider_specific_fields(message_delta["delta"])
|
||||
return finish_reason, usage, container, stop_reason_provider_specific_fields(message_delta["delta"])
|
||||
|
||||
def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -120,13 +120,14 @@ from litellm.utils import (
|
|||
)
|
||||
|
||||
from ..common_utils import (
|
||||
REFUSAL_STOP_FIELDS,
|
||||
STOP_REASON_FIELDS,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
process_anthropic_headers,
|
||||
refusal_provider_specific_fields,
|
||||
refusal_params,
|
||||
requires_native_compaction_beta,
|
||||
stop_reason_provider_specific_fields,
|
||||
strip_advisor_blocks_from_messages,
|
||||
tools_marked_for_a_tool_config_point,
|
||||
)
|
||||
|
|
@ -167,6 +168,8 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
|
|||
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
|
||||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
|
||||
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
|
||||
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
|
||||
|
|
@ -2919,28 +2922,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
web_search_results: Sequence[object],
|
||||
completion_response: Mapping[str, object],
|
||||
) -> list[object]:
|
||||
content: Final = completion_response.get("content")
|
||||
blocks: Final = content if isinstance(content, Sequence) else ()
|
||||
inputs: Final = {
|
||||
call_id: tool_input
|
||||
for block in blocks
|
||||
if isinstance(block, Mapping)
|
||||
and block.get("type") == "server_tool_use"
|
||||
and block.get("name") == "web_search"
|
||||
and isinstance((call_id := block.get("id")), str)
|
||||
and isinstance((tool_input := block.get("input")), Mapping)
|
||||
blocks: Final = _RESPONSE_BLOCKS.validate_python(completion_response.get("content") or ())
|
||||
results_by_id: Final = {
|
||||
tool_use_id: result
|
||||
for result in _RESPONSE_BLOCKS.validate_python(web_search_results)
|
||||
if result.get("type") == "web_search_tool_result"
|
||||
and isinstance((tool_use_id := result.get("tool_use_id")), str)
|
||||
}
|
||||
return [
|
||||
build_web_search_call(
|
||||
tool_id=tool_use_id,
|
||||
tool_input=inputs.get(tool_use_id, {}),
|
||||
result=result,
|
||||
tool_id=call_id,
|
||||
tool_input=block.get("input"),
|
||||
result=results_by_id.get(call_id, _PENDING_WEB_SEARCH_RESULT),
|
||||
status=None if call_id in results_by_id else "in_progress",
|
||||
)
|
||||
for result in web_search_results
|
||||
if isinstance(result, dict)
|
||||
and result.get("type") == "web_search_tool_result"
|
||||
and isinstance((tool_use_id := result.get("tool_use_id")), str)
|
||||
and tool_use_id in inputs
|
||||
for block in blocks
|
||||
if block.get("type") == "server_tool_use"
|
||||
and block.get("name") == "web_search"
|
||||
and isinstance((call_id := block.get("id")), str)
|
||||
]
|
||||
|
||||
def _build_provider_specific_fields(
|
||||
|
|
@ -2964,10 +2963,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
|
||||
if web_search_results is not None:
|
||||
provider_specific_fields["web_search_results"] = web_search_results
|
||||
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
|
||||
web_search_results,
|
||||
completion_response,
|
||||
)
|
||||
|
||||
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
|
||||
if web_search_results is not None or web_search_calls:
|
||||
provider_specific_fields["web_search_calls"] = web_search_calls
|
||||
|
||||
if tool_results is not None:
|
||||
provider_specific_fields["tool_results"] = tool_results
|
||||
|
|
@ -2989,10 +2988,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if compaction_blocks is not None:
|
||||
provider_specific_fields["compaction_blocks"] = compaction_blocks
|
||||
|
||||
refusal_fields: Final = refusal_provider_specific_fields(
|
||||
REFUSAL_STOP_FIELDS.validate_python(completion_response)
|
||||
stop_reason_fields: Final = stop_reason_provider_specific_fields(
|
||||
STOP_REASON_FIELDS.validate_python(completion_response)
|
||||
)
|
||||
return {**provider_specific_fields, **refusal_fields}
|
||||
return {**provider_specific_fields, **stop_reason_fields}
|
||||
|
||||
def transform_parsed_response(
|
||||
self,
|
||||
|
|
@ -3067,22 +3066,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if json_extra_content:
|
||||
merged_text = merged_text + json_extra_content if merged_text else json_extra_content
|
||||
|
||||
refusal_fields: Final = refusal_params(STOP_REASON_FIELDS.validate_python(completion_response))
|
||||
_message = litellm.Message(
|
||||
tool_calls=tool_calls_for_message,
|
||||
content=merged_text or None,
|
||||
provider_specific_fields=provider_specific_fields,
|
||||
thinking_blocks=thinking_blocks,
|
||||
reasoning_content=reasoning_content,
|
||||
**refusal_fields,
|
||||
)
|
||||
_message.provider_specific_fields = provider_specific_fields
|
||||
|
||||
if json_mode_message is not None:
|
||||
refusal_fields: Final = refusal_provider_specific_fields(
|
||||
REFUSAL_STOP_FIELDS.validate_python(completion_response)
|
||||
stop_reason_fields: Final = stop_reason_provider_specific_fields(
|
||||
STOP_REASON_FIELDS.validate_python(completion_response)
|
||||
)
|
||||
completion_response["stop_reason"] = "stop"
|
||||
if refusal_fields:
|
||||
json_mode_message.provider_specific_fields = dict(refusal_fields)
|
||||
if stop_reason_fields:
|
||||
json_mode_message.provider_specific_fields = dict(stop_reason_fields)
|
||||
if "refusal" in refusal_fields:
|
||||
json_mode_message.refusal = refusal_fields["refusal"]
|
||||
if thinking_blocks:
|
||||
json_mode_message.thinking_blocks = thinking_blocks
|
||||
json_mode_message.reasoning_content = reasoning_content
|
||||
|
|
|
|||
|
|
@ -2518,7 +2518,7 @@ def create_anthropic_model_list_response(
|
|||
}
|
||||
|
||||
|
||||
class RefusalStopFields(TypedDict, total=False):
|
||||
class StopReasonFields(TypedDict, total=False):
|
||||
stop_reason: ReadOnly[object]
|
||||
stop_details: ReadOnly[object]
|
||||
|
||||
|
|
@ -2535,16 +2535,38 @@ def _dict_or_empty(value: object) -> object:
|
|||
return value if _is_dict(value) else _no_stop_fields()
|
||||
|
||||
|
||||
REFUSAL_STOP_FIELDS: Final[TypeAdapter[RefusalStopFields]] = TypeAdapter(
|
||||
Annotated[RefusalStopFields, BeforeValidator(_dict_or_empty)]
|
||||
STOP_REASON_FIELDS: Final[TypeAdapter[StopReasonFields]] = TypeAdapter(
|
||||
Annotated[StopReasonFields, BeforeValidator(_dict_or_empty)]
|
||||
)
|
||||
|
||||
_STOP_REASONS_KEPT_RAW: Final = frozenset({"refusal", "pause_turn"})
|
||||
|
||||
def refusal_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
||||
def stop_reason_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
stop_reason: Final = message.get("stop_reason")
|
||||
stop_details: Final = message.get("stop_details")
|
||||
if stop_details is not None:
|
||||
return {"stop_reason": stop_reason, "stop_details": stop_details}
|
||||
if stop_reason == "refusal":
|
||||
if stop_reason in _STOP_REASONS_KEPT_RAW:
|
||||
return {"stop_reason": stop_reason}
|
||||
return {}
|
||||
|
||||
|
||||
class _RefusalStopDetails(TypedDict, total=False):
|
||||
explanation: ReadOnly[object]
|
||||
|
||||
|
||||
_REFUSAL_STOP_DETAILS: Final[TypeAdapter[_RefusalStopDetails]] = TypeAdapter(
|
||||
Annotated[_RefusalStopDetails, BeforeValidator(_dict_or_empty)]
|
||||
)
|
||||
|
||||
|
||||
class RefusalParams(TypedDict, total=False):
|
||||
refusal: ReadOnly[str]
|
||||
|
||||
|
||||
def refusal_params(message: Mapping[str, object]) -> RefusalParams:
|
||||
if message.get("stop_reason") != "refusal":
|
||||
return {}
|
||||
explanation: Final = _REFUSAL_STOP_DETAILS.validate_python(message.get("stop_details")).get("explanation")
|
||||
return {"refusal": explanation} if isinstance(explanation, str) and explanation else {}
|
||||
|
|
|
|||
|
|
@ -118,9 +118,10 @@ def _extract_proxy_litellm_metadata(
|
|||
|
||||
def _requested_router(kwargs: Mapping[str, object]) -> "Router | None":
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
requested: Final = kwargs.get("litellm_router")
|
||||
return requested if isinstance(requested, Router) else None
|
||||
return requested if isinstance(requested, (Router, PythonCore)) else None # pyright: ignore[reportReturnType] # the router passes its core, which carries Router's members
|
||||
|
||||
|
||||
def request_safeguards_evaluator(
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
|||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicStopDetails,
|
||||
AppliedEdit,
|
||||
CompactionBlock,
|
||||
ContentBlockDelta,
|
||||
|
|
@ -284,9 +285,7 @@ class _CombinedChunkSplitter:
|
|||
@staticmethod
|
||||
def _is_combined(chunk: "ModelResponseStream") -> bool:
|
||||
"""True if ``chunk`` carries response content AND a finish_reason."""
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
|
||||
|
||||
choices: Final = _optional_attr_sequence(chunk, "choices")
|
||||
if not choices:
|
||||
|
|
@ -302,7 +301,7 @@ class _CombinedChunkSplitter:
|
|||
or _optional_attr(delta, "tool_calls")
|
||||
or _optional_attr(delta, "reasoning_content")
|
||||
or _optional_attr(delta, "thinking_blocks")
|
||||
or openai_chat_refusal_text(delta)
|
||||
or refusal_reply_text(delta)
|
||||
)
|
||||
|
||||
_PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
|
||||
|
|
@ -659,6 +658,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
self.compaction_block = compaction_block
|
||||
self.iterations_usage = iterations_usage
|
||||
self._refusal_text: str = ""
|
||||
self._paused_turn: bool = False
|
||||
self._claude_refusal_stop_details: AnthropicStopDetails | None = None
|
||||
self.sent_compaction_block: bool = False
|
||||
# Per-phase flags so the compaction block's start/delta/stop events
|
||||
# are emitted (and the public state machine is advanced) in
|
||||
|
|
@ -953,7 +954,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
current_content_block_index=self.current_content_block_index,
|
||||
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
|
||||
)
|
||||
processed_chunk = self._with_refusal_stop_details(processed_chunk)
|
||||
processed_chunk = self._with_stop_reason_fields(processed_chunk)
|
||||
|
||||
# Check if this is a usage chunk and we have a held stop_reason chunk
|
||||
if will_merge_into_held:
|
||||
|
|
@ -1188,7 +1189,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
current_content_block_index=self.current_content_block_index,
|
||||
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
|
||||
)
|
||||
processed_chunk = self._with_refusal_stop_details(processed_chunk)
|
||||
processed_chunk = self._with_stop_reason_fields(processed_chunk)
|
||||
|
||||
# Check if this is a usage chunk and we have a held stop_reason chunk
|
||||
if will_merge_into_held:
|
||||
|
|
@ -1406,14 +1407,22 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
def _increment_content_block_index(self):
|
||||
self.current_content_block_index += 1
|
||||
|
||||
def _with_refusal_stop_details(
|
||||
def _with_stop_reason_fields(
|
||||
self,
|
||||
processed_chunk: ContentBlockDelta | MessageBlockDelta,
|
||||
) -> ContentBlockDelta | MessageBlockDelta:
|
||||
if processed_chunk["type"] != "message_delta" or not self._refusal_text:
|
||||
if processed_chunk["type"] != "message_delta":
|
||||
return processed_chunk
|
||||
delta: Final = processed_chunk["delta"]
|
||||
if delta.get("stop_reason") == "max_tokens":
|
||||
if self._paused_turn:
|
||||
paused_delta: Final[MessageDelta] = {**delta, "stop_reason": "pause_turn"}
|
||||
paused_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": paused_delta}
|
||||
return paused_chunk
|
||||
if delta.get("stop_reason") == "refusal" and self._claude_refusal_stop_details is not None:
|
||||
claude_refusal_delta: Final[MessageDelta] = {**delta, "stop_details": self._claude_refusal_stop_details}
|
||||
claude_refusal_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": claude_refusal_delta}
|
||||
return claude_refusal_chunk
|
||||
if not self._refusal_text or delta.get("stop_reason") == "max_tokens":
|
||||
return processed_chunk
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
refusal_stop_details,
|
||||
|
|
@ -1470,9 +1479,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
@staticmethod
|
||||
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
|
||||
from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
|
||||
|
||||
choice: Final = chunk.choices[0]
|
||||
if choice.finish_reason is not None:
|
||||
|
|
@ -1482,7 +1489,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
return False
|
||||
if getattr(delta, "content", None):
|
||||
return False
|
||||
if openai_chat_refusal_text(delta):
|
||||
if refusal_reply_text(delta):
|
||||
return False
|
||||
if getattr(delta, "reasoning_content", None):
|
||||
return False
|
||||
|
|
@ -1497,6 +1504,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
return False
|
||||
return True
|
||||
|
||||
def _record_anthropic_stop_fields(self, delta: object) -> None:
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
anthropic_refusal_stop_details,
|
||||
anthropic_stop_reason,
|
||||
)
|
||||
|
||||
if anthropic_stop_reason(delta) == "pause_turn":
|
||||
self._paused_turn = True
|
||||
claude_refusal_stop_details: Final = anthropic_refusal_stop_details(delta)
|
||||
if claude_refusal_stop_details is not None:
|
||||
self._claude_refusal_stop_details = claude_refusal_stop_details
|
||||
|
||||
def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
|
||||
"""
|
||||
Determine if we should start a new content block based on the processed chunk.
|
||||
|
|
@ -1507,16 +1526,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
- Different content types in the response
|
||||
- Specific markers in the content
|
||||
"""
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
|
||||
|
||||
from .transformation import LiteLLMAnthropicMessagesAdapter
|
||||
|
||||
self._record_anthropic_stop_fields(chunk.choices[0].delta)
|
||||
if chunk.choices[0].finish_reason is not None:
|
||||
return False
|
||||
|
||||
refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
|
||||
refusal_text: Final = refusal_reply_text(chunk.choices[0].delta)
|
||||
if refusal_text is not None:
|
||||
self._refusal_text = self._refusal_text + refusal_text
|
||||
|
||||
|
|
|
|||
|
|
@ -150,7 +150,9 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import (
|
|||
is_system_role_message,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import (
|
||||
openai_chat_refusal_text,
|
||||
anthropic_refusal_stop_details,
|
||||
anthropic_stop_reason,
|
||||
refusal_reply_text,
|
||||
refusal_stop_details,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -398,6 +400,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
return None
|
||||
|
||||
def _streamed_tool_call_signature(self, tool_call_id: str, tool_call: object) -> str | None:
|
||||
id_parts: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
|
||||
return id_parts[1] if len(id_parts) > 1 else self._extract_signature_from_tool_call(tool_call)
|
||||
|
||||
def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None:
|
||||
"""
|
||||
Extract signature from a tool_use content block's provider_specific_fields.
|
||||
|
|
@ -1467,7 +1473,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
new_content.append(
|
||||
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
|
||||
)
|
||||
if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
|
||||
if (refusal_text := refusal_reply_text(choice.message)) is not None:
|
||||
new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
|
||||
# Handle tool calls (in parallel to text content)
|
||||
if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
|
||||
|
|
@ -1634,7 +1640,15 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_name_mapping=tool_name_mapping,
|
||||
)
|
||||
refusal_text: Final = next(
|
||||
(text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
|
||||
(text for choice in response.choices if (text := refusal_reply_text(choice.message)) is not None),
|
||||
None,
|
||||
)
|
||||
claude_refusal_stop_details: Final = next(
|
||||
(
|
||||
details
|
||||
for choice in response.choices
|
||||
if (details := anthropic_refusal_stop_details(choice.message)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
|
@ -1643,8 +1657,10 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
|
||||
## extract finish reason
|
||||
openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop"
|
||||
translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
|
||||
openai_finish_reason=openai_finish_reason
|
||||
translated_finish_reason: Final = (
|
||||
"pause_turn"
|
||||
if response.choices and anthropic_stop_reason(response.choices[0].message) == "pause_turn"
|
||||
else self._translate_openai_finish_reason_to_anthropic(openai_finish_reason=openai_finish_reason)
|
||||
)
|
||||
anthropic_finish_reason: Final = (
|
||||
"compaction"
|
||||
|
|
@ -1686,7 +1702,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
usage=anthropic_usage,
|
||||
content=anthropic_content,
|
||||
stop_reason=anthropic_finish_reason,
|
||||
stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
|
||||
stop_details=(
|
||||
(claude_refusal_stop_details or refusal_stop_details(refusal_text))
|
||||
if anthropic_finish_reason == "refusal"
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
|
||||
|
|
@ -1712,10 +1732,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
):
|
||||
raw_id = choice.delta.tool_calls[0].id or str(uuid.uuid4())
|
||||
tool_name = choice.delta.tool_calls[0].function.name or ""
|
||||
thought_sig: str | None = None
|
||||
if THOUGHT_SIGNATURE_SEPARATOR in raw_id:
|
||||
parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
|
||||
thought_sig = parts[1] if len(parts) > 1 else None
|
||||
thought_sig = self._streamed_tool_call_signature(raw_id, choice.delta.tool_calls[0])
|
||||
tool_block: dict[str, object] = {
|
||||
"type": "tool_use",
|
||||
"id": normalize_anthropic_tool_use_id(raw_id),
|
||||
|
|
@ -1727,7 +1744,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
"signature": thought_sig,
|
||||
}
|
||||
return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
|
||||
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
|
||||
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or refusal_reply_text(
|
||||
choice.delta
|
||||
) is not None:
|
||||
return "text", TextBlock(type="text", text="")
|
||||
|
|
@ -1810,7 +1827,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
|
||||
else:
|
||||
refusal_text: Final = "".join(
|
||||
refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
|
||||
refusal for choice in choices if (refusal := refusal_reply_text(choice.delta)) is not None
|
||||
)
|
||||
return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
|
|||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Final, cast, get_type_hints
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config
|
||||
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
|
|
@ -162,6 +162,34 @@ def openai_chat_refusal_text(message_or_delta: object) -> str | None:
|
|||
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
|
||||
|
||||
|
||||
def anthropic_stop_reason(message_or_delta: object) -> str | None:
|
||||
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "stop_reason")
|
||||
|
||||
|
||||
def refusal_reply_text(message_or_delta: object) -> str | None:
|
||||
if anthropic_stop_reason(message_or_delta) == "refusal":
|
||||
return None
|
||||
return openai_chat_refusal_text(message_or_delta)
|
||||
|
||||
|
||||
@with_config(ConfigDict(extra="allow"))
|
||||
class _AnthropicRefusalStopDetails(AnthropicStopDetails, total=False):
|
||||
pass
|
||||
|
||||
|
||||
_ANTHROPIC_REFUSAL_STOP_DETAILS: Final = TypeAdapter(_AnthropicRefusalStopDetails)
|
||||
|
||||
|
||||
def anthropic_refusal_stop_details(message_or_delta: object) -> AnthropicStopDetails | None:
|
||||
provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None)
|
||||
if _mapping_str_field(provider_specific_fields, "stop_reason") != "refusal":
|
||||
return None
|
||||
try:
|
||||
return _ANTHROPIC_REFUSAL_STOP_DETAILS.validate_python(_mapping_field(provider_specific_fields, "stop_details"))
|
||||
except ValidationError:
|
||||
return refusal_stop_details(openai_chat_refusal_text(message_or_delta))
|
||||
|
||||
|
||||
def _responses_message_refusal_text(item: object) -> str | None:
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
|
||||
|
||||
|
|
|
|||
|
|
@ -155,6 +155,8 @@ def collapse_pure_text_chunks(
|
|||
continue
|
||||
|
||||
json_str = ev[idx + len("data:") :].strip()
|
||||
if not json_str.startswith("{"):
|
||||
continue
|
||||
try:
|
||||
data = json.loads(json_str)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
|
|
@ -206,6 +208,11 @@ def collapse_pure_text_chunks(
|
|||
return out
|
||||
|
||||
|
||||
def _has_non_object_data(event_str: str) -> bool:
|
||||
data_start: Final = event_str.find("data:")
|
||||
return data_start != -1 and not event_str[data_start + len("data:") :].lstrip().startswith("{")
|
||||
|
||||
|
||||
def build_complete_streaming_response_legacy(
|
||||
all_chunks: Sequence[str | bytes],
|
||||
litellm_logging_obj: LiteLLMLoggingObj,
|
||||
|
|
@ -243,6 +250,8 @@ def build_complete_streaming_response_legacy(
|
|||
# text payload happens to contain "[DONE]" is not dropped.
|
||||
if any(line.strip() == "data: [DONE]" for line in event_str.split("\n")):
|
||||
continue
|
||||
if _has_non_object_data(event_str):
|
||||
continue
|
||||
transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk(
|
||||
chunk=event_str
|
||||
)
|
||||
|
|
@ -268,21 +277,25 @@ def build_complete_streaming_response_legacy(
|
|||
return complete_streaming_response
|
||||
|
||||
|
||||
def extract_sse_data(event_str: str) -> dict | None:
|
||||
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
|
||||
def _sse_data_payload(event_str: str) -> str | None:
|
||||
for line in split_sse_lines(event_str):
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("data:"):
|
||||
payload = stripped[len("data:") :].strip()
|
||||
if not payload or payload == "[DONE]":
|
||||
return None
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return stripped[len("data:") :].strip()
|
||||
return None
|
||||
|
||||
|
||||
def extract_sse_data(event_str: str) -> dict | None:
|
||||
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
|
||||
payload: Final = _sse_data_payload(event_str)
|
||||
if payload is None or not payload.startswith("{"):
|
||||
return None
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def build_usage_only_response_from_chunks(
|
||||
all_chunks: Sequence[str | bytes],
|
||||
model: str,
|
||||
|
|
@ -415,27 +428,81 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset(
|
|||
)
|
||||
|
||||
|
||||
_ERROR_PAYLOAD_KEYS: Final = frozenset({"type", "error", "request_id"})
|
||||
|
||||
|
||||
def _is_anthropic_event_type(event_type: object) -> bool:
|
||||
return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
|
||||
|
||||
|
||||
def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool:
|
||||
return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks)
|
||||
|
||||
|
||||
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
|
||||
raw: Final = b"".join(
|
||||
def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes:
|
||||
return b"".join(
|
||||
chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
|
||||
for chunk in all_chunks
|
||||
if isinstance(chunk, (str, bytes))
|
||||
)
|
||||
|
||||
|
||||
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
|
||||
try:
|
||||
return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False)
|
||||
return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False)
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _sse_event_data(event: str) -> Mapping[str, object] | None:
|
||||
return as_str_mapping(extract_sse_data(event))
|
||||
|
||||
|
||||
_JSON_ESCAPED_LINE_BREAKS: Final = MappingProxyType(
|
||||
str.maketrans({"\x85": "\\u0085", "\u2028": "\\u2028", "\u2029": "\\u2029"})
|
||||
)
|
||||
|
||||
|
||||
def _sse_frames(sse_stream: str) -> Sequence[str]:
|
||||
return "\n".join(split_sse_lines(sse_stream)).split("\n\n")
|
||||
|
||||
|
||||
def _frames_of_the_widest_reader(sse_stream: str) -> Sequence[str]:
|
||||
"""The stream's frames as httpx's ``str.splitlines`` reads them, past a form feed or U+2028 in any line."""
|
||||
return "\n".join(sse_stream.splitlines()).split("\n\n")
|
||||
|
||||
|
||||
def _sse_field(line: str) -> tuple[str, str]:
|
||||
name, _, value = line.partition(":")
|
||||
return name, value.removeprefix(" ")
|
||||
|
||||
|
||||
def _sse_fields(frame: str) -> tuple[tuple[str, str], ...]:
|
||||
"""The frame's fields as a client reads them: a name matches exactly and a comment is no field."""
|
||||
return tuple(_sse_field(line) for line in frame.split("\n") if line and not line.startswith(":"))
|
||||
|
||||
|
||||
def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
|
||||
match fields:
|
||||
case () | (("event", _),) | (("data", "[DONE]"),):
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _json_mapping(payload: str) -> Mapping[str, object] | None:
|
||||
try:
|
||||
value: Final[object] = json.loads(payload)
|
||||
except ValueError:
|
||||
return None
|
||||
return as_str_mapping(value)
|
||||
|
||||
|
||||
def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
event_data
|
||||
for event in split_sse_chunk_into_events(sse_stream)
|
||||
if (event_data := extract_sse_data(event)) is not None
|
||||
if (event_data := _sse_event_data(event)) is not None
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -444,7 +511,8 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
|
|||
(
|
||||
message
|
||||
for event_data in _parsed_sse_events(sse_stream)
|
||||
if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict)
|
||||
if event_data.get("type") == "message_start"
|
||||
and (message := as_str_mapping(event_data.get("message"))) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
|
@ -482,11 +550,151 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
|
|||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return False
|
||||
return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream))
|
||||
return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream))
|
||||
|
||||
|
||||
def _is_json_no_reader_can_take(payload: str) -> bool:
|
||||
try:
|
||||
value: Final[object] = json.loads(payload)
|
||||
except ValueError:
|
||||
return False
|
||||
if value is None:
|
||||
return False
|
||||
event: Final = as_str_mapping(value)
|
||||
return event is None or isinstance(event.get("type"), (list, dict))
|
||||
|
||||
|
||||
def has_unreadable_json_event(all_chunks: Sequence[object]) -> bool:
|
||||
"""Whether a frame's data is JSON that no reader of these streams can take as an event."""
|
||||
sse_stream: Final = codecs.getincrementaldecoder("utf-8")(errors="replace").decode(
|
||||
_joined_sse_bytes(all_chunks), final=False
|
||||
)
|
||||
return any(_is_json_no_reader_can_take(_sse_data_payload(event) or "") for event in _sse_frames(sse_stream))
|
||||
|
||||
|
||||
def anthropic_sse_event_type(event: str) -> object:
|
||||
return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None)
|
||||
|
||||
|
||||
def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
|
||||
"""The string ``type`` of every event the stream sent."""
|
||||
event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
return frozenset(event_type for event_type in event_types if isinstance(event_type, str))
|
||||
|
||||
|
||||
def _is_named_error_event(event: str) -> bool:
|
||||
names: Final = tuple(value for name, value in _sse_fields(event) if name == "event")
|
||||
return names[-1:] == ("error",)
|
||||
|
||||
|
||||
def _is_error_event(event: str) -> bool:
|
||||
event_type: Final = anthropic_sse_event_type(event)
|
||||
return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type))
|
||||
|
||||
|
||||
def _frames_a_client_dispatches(sse_stream: str) -> Sequence[str]:
|
||||
return "\n".join((*split_sse_lines(sse_stream), "")).split("\n\n")[:-1]
|
||||
|
||||
|
||||
def _error_events(sse_stream: str) -> tuple[str, ...]:
|
||||
return tuple(event for event in _frames_a_client_dispatches(sse_stream) if _is_error_event(event))
|
||||
|
||||
|
||||
def _plain_error_event(event: str) -> str:
|
||||
payload: Final = _sse_data_payload(event) or ""
|
||||
one_line: Final = payload.translate(_JSON_ESCAPED_LINE_BREAKS) if _json_mapping(payload) is not None else payload
|
||||
return "\n".join(("event: error", *(f"data: {line}" for line in one_line.splitlines()[:1])))
|
||||
|
||||
|
||||
def anthropic_sse_error_events(all_chunks: Sequence[object]) -> tuple[str, ...]:
|
||||
"""The ``error`` events of the stream, each as an ``event:`` line and the ``data:`` line assembly read.
|
||||
|
||||
A stream whose upstream fails after it started ends with an ``error`` event in place of
|
||||
``message_stop``. That frame is the outcome the client is owed, not content, so assembly through
|
||||
the upstream error leaves it out and a caller re-emitting a rewritten response puts it back,
|
||||
without any other line of the frame, since that is content no guardrail read.
|
||||
A frame counts by its data ``type``, or by its SSE ``event:`` name unless its data is another
|
||||
Anthropic event, since a client SDK raises on the name alone and an upstream that is not
|
||||
Anthropic may send anything as the data.
|
||||
"""
|
||||
return tuple(_plain_error_event(event) for event in _error_events(_joined_sse_stream(all_chunks) or ""))
|
||||
|
||||
|
||||
def anthropic_sse_stopped_block_index(event: str) -> object:
|
||||
return next(
|
||||
(
|
||||
event_data.get("index")
|
||||
for event_data in _parsed_sse_events(event)
|
||||
if event_data.get("type") == "content_block_stop"
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _block_index(event_data: Mapping[str, object]) -> int | None:
|
||||
index: Final = event_data.get("index")
|
||||
return index if isinstance(index, int) else None
|
||||
|
||||
|
||||
def _started_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
|
||||
starts: Final = (event for event in events if event.get("type") == "content_block_start")
|
||||
return MappingProxyType(
|
||||
{
|
||||
index: block
|
||||
for event in starts
|
||||
if (index := _block_index(event)) is not None
|
||||
and (block := as_str_mapping(event.get("content_block"))) is not None
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def anthropic_sse_started_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
|
||||
"""The ``content_block`` each ``content_block_start`` of the stream opened, by block index."""
|
||||
return _started_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
|
||||
|
||||
def _unfinished_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
|
||||
stopped: Final = frozenset(_block_index(event) for event in events if event.get("type") == "content_block_stop")
|
||||
return MappingProxyType({index: block for index, block in _started_blocks(events).items() if index not in stopped})
|
||||
|
||||
|
||||
def anthropic_sse_unfinished_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
|
||||
"""The blocks the stream opened and never stopped, which only a failed upstream leaves."""
|
||||
return _unfinished_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
|
||||
|
||||
|
||||
def _sse_events_with_data(sse_stream: str) -> tuple[tuple[str, Mapping[str, object]], ...]:
|
||||
return tuple(
|
||||
(event, event_data)
|
||||
for event in split_sse_chunk_into_events(sse_stream)
|
||||
if (event_data := _sse_event_data(event)) is not None
|
||||
)
|
||||
|
||||
|
||||
def _content_events_before_upstream_error(sse_stream: str) -> tuple[str, ...]:
|
||||
"""The stream's events minus its upstream ``error`` and every cut-off block assembly cannot read.
|
||||
|
||||
A cut-off text block is still text the client was sent, so it stays. A cut-off tool call's
|
||||
input is a JSON prefix, and a cut-off thinking block has no signature, so neither is a block the
|
||||
client can act on, and both are left out.
|
||||
"""
|
||||
if not _error_events(sse_stream):
|
||||
return (sse_stream,)
|
||||
events: Final = _sse_events_with_data(sse_stream)
|
||||
cut_off: Final = frozenset(
|
||||
index
|
||||
for index, block in _unfinished_blocks(tuple(event_data for _, event_data in events)).items()
|
||||
if block.get("type") != "text"
|
||||
)
|
||||
return tuple(
|
||||
event
|
||||
for event, event_data in events
|
||||
if event_data.get("type") != "error" and _block_index(event_data) not in cut_off
|
||||
)
|
||||
|
||||
|
||||
def assemble_anthropic_sse_stream(
|
||||
all_chunks: Sequence[object], *, restore_identity: bool = False
|
||||
all_chunks: Sequence[object], *, restore_identity: bool = False, through_upstream_error: bool = False
|
||||
) -> ModelResponse | None:
|
||||
"""Assemble raw Anthropic SSE frames into a ModelResponse.
|
||||
|
||||
|
|
@ -494,6 +702,11 @@ def assemble_anthropic_sse_stream(
|
|||
assembler does not carry through. It is off by default so callers that re-emit the assembled
|
||||
response keep the wire shape they had before this helper was shared. The writes land on a
|
||||
freshly built object that is unreachable from caller state until returned.
|
||||
|
||||
A stream whose upstream failed after it started ends with an ``error`` event in place of
|
||||
``message_stop`` and assembles to None, unless ``through_upstream_error`` asks for the content
|
||||
the client was sent before the failure. That is for a guardrail that scans the stream and then
|
||||
passes the upstream error on; a caller recording what was served keeps None.
|
||||
"""
|
||||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
|
|
@ -502,9 +715,12 @@ def assemble_anthropic_sse_stream(
|
|||
if message_start is None:
|
||||
return None
|
||||
model: Final = message_start.get("model") if restore_identity else None
|
||||
content_events: Final = (
|
||||
_content_events_before_upstream_error(sse_stream) if through_upstream_error else (sse_stream,)
|
||||
)
|
||||
try:
|
||||
assembled: Final = build_complete_streaming_response(
|
||||
all_chunks=(sse_stream,),
|
||||
all_chunks=content_events,
|
||||
litellm_logging_obj=None, # pyright: ignore[reportArgumentType] # only forwarded to stream_chunk_builder, which accepts None
|
||||
model=model if isinstance(model, str) else "",
|
||||
)
|
||||
|
|
@ -522,6 +738,21 @@ def assemble_anthropic_sse_stream(
|
|||
return assembled
|
||||
|
||||
|
||||
def _is_error_payload(payload: str) -> bool:
|
||||
event: Final = _json_mapping(payload)
|
||||
if event is None or not _ERROR_PAYLOAD_KEYS.issuperset(event):
|
||||
return False
|
||||
return event.get("type") == "error" or ("type" not in event and isinstance(event.get("error"), Mapping))
|
||||
|
||||
|
||||
def _is_error_frame(fields: Sequence[tuple[str, str]]) -> bool:
|
||||
match fields:
|
||||
case (("data", payload),) | (("event", "error"), ("data", payload)):
|
||||
return _is_error_payload(payload)
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
|
||||
"""Whether the buffered stream carries nothing but error frames.
|
||||
|
||||
|
|
@ -537,7 +768,6 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
|
|||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return False
|
||||
events: Final = _parsed_sse_events(sse_stream)
|
||||
return len(events) > 0 and all(
|
||||
event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
|
||||
)
|
||||
frames: Final = tuple(_sse_fields(frame) for frame in _frames_of_the_widest_reader(sse_stream))
|
||||
with_data: Final = tuple(fields for fields in frames if not _carries_no_data(fields))
|
||||
return len(with_data) > 0 and all(_is_error_frame(fields) for fields in with_data)
|
||||
|
|
|
|||
|
|
@ -544,6 +544,17 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
|
||||
return AgentCoreParsedResponse(content=content, usage=usage_data, final_message=final_message)
|
||||
|
||||
@staticmethod
|
||||
def _message_stop_chunk(model: str) -> ModelResponseStream:
|
||||
chunk: Final = ModelResponseStream(
|
||||
id=f"chatcmpl-{uuid.uuid4()}",
|
||||
created=0,
|
||||
model=model,
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
chunk.choices = [StreamingChoices(finish_reason="stop", index=0, delta=Delta())]
|
||||
return chunk
|
||||
|
||||
def _stream_agentcore_response_sync(
|
||||
self,
|
||||
response: httpx.Response,
|
||||
|
|
@ -553,6 +564,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
Internal sync generator that parses SSE and yields ModelResponse chunks.
|
||||
"""
|
||||
buffer = ""
|
||||
sent_finish = False # rebind-ok: flips when a finish chunk goes out
|
||||
message_stopped = False # rebind-ok: flips on messageStop, the only end some runtimes send
|
||||
for text_chunk in response.iter_text():
|
||||
buffer += text_chunk
|
||||
|
||||
|
|
@ -624,7 +637,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
total_tokens=usage_data.get("totalTokens", 0),
|
||||
),
|
||||
)
|
||||
sent_finish = True
|
||||
yield chunk
|
||||
message_stopped |= "messageStop" in event_payload
|
||||
|
||||
# Process final message
|
||||
if "message" in data_obj and isinstance(data_obj["message"], dict):
|
||||
|
|
@ -641,12 +656,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
delta=Delta(),
|
||||
)
|
||||
]
|
||||
sent_finish = True
|
||||
yield chunk
|
||||
|
||||
except json.JSONDecodeError:
|
||||
verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
|
||||
continue
|
||||
|
||||
if message_stopped and not sent_finish:
|
||||
yield self._message_stop_chunk(model)
|
||||
|
||||
def get_sync_custom_stream_wrapper(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -775,6 +794,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
Internal async generator that parses SSE and yields ModelResponse chunks.
|
||||
"""
|
||||
buffer = ""
|
||||
sent_finish = False # rebind-ok: flips when a finish chunk goes out
|
||||
message_stopped = False # rebind-ok: flips on messageStop, the only end some runtimes send
|
||||
async for text_chunk in response.aiter_text():
|
||||
buffer += text_chunk
|
||||
|
||||
|
|
@ -846,7 +867,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
total_tokens=usage_data.get("totalTokens", 0),
|
||||
),
|
||||
)
|
||||
sent_finish = True
|
||||
yield chunk
|
||||
message_stopped |= "messageStop" in event_payload
|
||||
|
||||
# Process final message
|
||||
if "message" in data_obj and isinstance(data_obj["message"], dict):
|
||||
|
|
@ -863,12 +886,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
|
|||
delta=Delta(),
|
||||
)
|
||||
]
|
||||
sent_finish = True
|
||||
yield chunk
|
||||
|
||||
except json.JSONDecodeError:
|
||||
verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
|
||||
continue
|
||||
|
||||
if message_stopped and not sent_finish:
|
||||
yield self._message_stop_chunk(model)
|
||||
|
||||
async def get_async_custom_stream_wrapper(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -131,6 +131,8 @@ from litellm.responses.streaming_iterator import (
|
|||
ResponsesAPIStreamingIterator,
|
||||
ResponsesWebSocketStreaming,
|
||||
SyncResponsesAPIStreamingIterator,
|
||||
TokenLimitCallback,
|
||||
with_token_limit_checks,
|
||||
)
|
||||
from litellm.types.containers.main import (
|
||||
ContainerFileListResponse,
|
||||
|
|
@ -598,10 +600,10 @@ def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
|
|||
)
|
||||
|
||||
|
||||
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
|
||||
"""Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
|
||||
enforcement, so the Responses WebSocket loop can charge every
|
||||
``response.create`` frame, not just the connection's first one.
|
||||
def _ws_callbacks_exposing(method_name: str) -> tuple[object, ...]:
|
||||
"""Duck-type discover proxy hooks exposing a per-frame check, so the
|
||||
Responses WebSocket loop can run it on every ``response.create`` frame,
|
||||
not just the connection's first one.
|
||||
|
||||
Uses duck-typing on ``litellm.callbacks`` (rather than importing the
|
||||
proxy hook directly) to avoid a layering violation (SDK importing from
|
||||
|
|
@ -612,13 +614,28 @@ def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
|
|||
callbacks: Final = cast( # cast-ok: callback registry is inspected before protocol use
|
||||
Sequence[object], _litellm.callbacks
|
||||
)
|
||||
return tuple(callback for callback in callbacks if callable(getattr(callback, method_name, None)))
|
||||
|
||||
|
||||
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
|
||||
return tuple(
|
||||
cast(ProjectQuotaCallback, callback) # cast-ok: required callback method is callable
|
||||
for callback in callbacks
|
||||
if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
|
||||
for callback in _ws_callbacks_exposing("enforce_project_io_token_quota_for_frame")
|
||||
)
|
||||
|
||||
|
||||
def _collect_ws_token_limit_callbacks() -> tuple[TokenLimitCallback, ...]:
|
||||
return tuple(
|
||||
cast(TokenLimitCallback, callback) # cast-ok: required callback method is callable
|
||||
for callback in _ws_callbacks_exposing("enforce_token_limits_for_frame")
|
||||
)
|
||||
|
||||
|
||||
def _ws_model_group(litellm_metadata: Mapping[str, object] | None, model: str) -> str:
|
||||
model_group: Final = (litellm_metadata or {}).get("model_group")
|
||||
return model_group if isinstance(model_group, str) and model_group else model
|
||||
|
||||
|
||||
def _unwrap_provider_error(error: Exception) -> Exception:
|
||||
if not isinstance(error, httpx.TransportError):
|
||||
return error
|
||||
|
|
@ -7285,6 +7302,12 @@ class BaseLLMHTTPHandler:
|
|||
- Forwards events over the websocket connection
|
||||
"""
|
||||
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
|
||||
_ws_turn_admission: Final = with_token_limit_checks(
|
||||
turn_admission,
|
||||
_collect_ws_token_limit_callbacks(),
|
||||
user_api_key_dict,
|
||||
_ws_model_group(litellm_metadata, model),
|
||||
)
|
||||
|
||||
_ws_guardrail_callbacks: list = []
|
||||
_ws_output_guardrail_callbacks: list = []
|
||||
|
|
@ -7332,7 +7355,7 @@ class BaseLLMHTTPHandler:
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
first_message=first_message,
|
||||
quota_callbacks=_ws_quota_callbacks,
|
||||
turn_admission=turn_admission,
|
||||
turn_admission=_ws_turn_admission,
|
||||
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
|
||||
frame_guard=frame_guard,
|
||||
**kwargs,
|
||||
|
|
@ -7429,7 +7452,7 @@ class BaseLLMHTTPHandler:
|
|||
authorized_model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
request_defaults=request_defaults,
|
||||
turn_admission=turn_admission,
|
||||
turn_admission=_ws_turn_admission,
|
||||
frame_guard=frame_guard,
|
||||
)
|
||||
return await streaming.bidirectional_forward()
|
||||
|
|
|
|||
|
|
@ -1,16 +1,21 @@
|
|||
import json
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from urllib.parse import parse_qs, urlparse, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL
|
||||
from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL, DEEPGRAM_SPEAK_DEFAULT_MODEL
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import WebSocketSessionUsage
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
|
||||
|
||||
_WEBSOCKET_SCHEMES: Final = MappingProxyType({"https": "wss", "http": "ws", "wss": "wss", "ws": "ws"})
|
||||
DEEPGRAM_LISTEN_CALLBACK_PARAMS: Final = frozenset({"callback", "callback_method"})
|
||||
|
|
@ -199,3 +204,71 @@ def deepgram_listen_transcript(websocket_messages: Sequence[Mapping[str, object]
|
|||
if frame.get("type") == "Results"
|
||||
if (transcript := _final_transcript(frame)) is not None
|
||||
)
|
||||
|
||||
|
||||
_AURA_GENERATION: Final = re.compile(r"aura(?:-\d+)?(?=-[a-z])")
|
||||
|
||||
|
||||
def deepgram_speak_websocket_target(api_base: str | None, query_string: str) -> str:
|
||||
speak_url: Final = urlsplit(join_api_base(api_base or DEEPGRAM_DEFAULT_API_BASE, "speak"))
|
||||
scheme: Final = _WEBSOCKET_SCHEMES.get(speak_url.scheme, speak_url.scheme)
|
||||
params: Final = _first_occurrences(query_string)
|
||||
query: Final = params if params.get("model") else params.remove("model").add("model", DEEPGRAM_SPEAK_DEFAULT_MODEL)
|
||||
return urlunsplit((scheme, speak_url.netloc, speak_url.path, str(query), ""))
|
||||
|
||||
|
||||
def deepgram_speak_model(upstream_url: str) -> str:
|
||||
models: Final = parse_qs(urlparse(upstream_url).query).get("model")
|
||||
return models[0] if models else DEEPGRAM_SPEAK_DEFAULT_MODEL
|
||||
|
||||
|
||||
def deepgram_speak_requested_model(query_string: str) -> str:
|
||||
"""Read off the upstream target, so the voice a key is authorized for is the voice Deepgram receives"""
|
||||
return deepgram_speak_model(deepgram_speak_websocket_target(None, query_string))
|
||||
|
||||
|
||||
def deepgram_speak_registry_key(upstream_url: str) -> str:
|
||||
"""Deepgram bills every voice of an Aura generation at one rate, so ``aura-2-thalia-en`` is priced at the
|
||||
``deepgram/aura-2`` row"""
|
||||
model: Final = deepgram_speak_model(upstream_url)
|
||||
generation: Final = _AURA_GENERATION.match(model)
|
||||
return f"{LlmProviders.DEEPGRAM.value}/{generation.group() if generation else model}"
|
||||
|
||||
|
||||
def deepgram_speak_character_rate(upstream_url: str) -> float | None:
|
||||
"""Only an exact registry hit counts, so a voice is never billed at a neighbouring row's rate"""
|
||||
registry_key: Final = deepgram_speak_registry_key(upstream_url)
|
||||
model_info: Final = lookup_model_info_for_registration(registry_key, LlmProviders.DEEPGRAM.value)
|
||||
if isinstance(model_info, ModelInfoLookupMiss) or model_info["key"] != registry_key:
|
||||
return None
|
||||
return model_info.get("input_cost_per_character")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeepgramSpeakMeter:
|
||||
"""Holds the rate checked when the socket opened, so a cost-map reload mid-session can't change or drop the price"""
|
||||
|
||||
upstream_url: str
|
||||
character_rate: float
|
||||
|
||||
def measure(self, client_text: str) -> int:
|
||||
"""Counts a Speak message's text as it is sent. Deepgram also speaks, and bills, unflushed text on its own (a
|
||||
long buffer, Close, a close frame), so counting only flushed text would let a client get audio free, while
|
||||
counting on send can only over-bill text Deepgram discards, such as a Clear or an oversized Speak. Never
|
||||
raises, so a malformed frame can't end the session"""
|
||||
try:
|
||||
message: Final[object] = json.loads(client_text)
|
||||
except (ValueError, RecursionError):
|
||||
return 0
|
||||
match message:
|
||||
case {"type": "Speak", "text": str() as text}:
|
||||
return len(text)
|
||||
case _:
|
||||
return 0
|
||||
|
||||
def price(self, total: int) -> WebSocketSessionUsage:
|
||||
return WebSocketSessionUsage(
|
||||
model=deepgram_speak_model(self.upstream_url),
|
||||
custom_llm_provider=LlmProviders.DEEPGRAM.value,
|
||||
response_cost=total * self.character_rate,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -41,8 +41,9 @@ class LangGraphSSEStreamIterator:
|
|||
return self
|
||||
|
||||
def __aiter__(self):
|
||||
"""Initialize async iteration."""
|
||||
self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
|
||||
"""Initialize async iteration once; CustomStreamWrapper re-enters `async for` on every chunk."""
|
||||
if self.async_line_iterator is None:
|
||||
self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
|
||||
return self
|
||||
|
||||
def _parse_sse_line(self, line: str) -> ModelResponseStream | None:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,11 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
|
||||
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
merge_consecutive_system_messages,
|
||||
messages_with_file_parts_as_image_urls,
|
||||
system_messages_first,
|
||||
)
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
|
|
@ -693,14 +697,18 @@ class OCIChatConfig(BaseConfig):
|
|||
chatRequest=chat_request,
|
||||
)
|
||||
else:
|
||||
generic_messages: Final = list(messages_with_file_parts_as_image_urls(messages))
|
||||
normalized_messages: Final = (
|
||||
merge_consecutive_system_messages(system_messages_first(generic_messages))
|
||||
if model.lower().removeprefix("oci/").startswith("google.gemini")
|
||||
else generic_messages
|
||||
)
|
||||
data = OCICompletionPayload(
|
||||
compartmentId=oci_compartment_id,
|
||||
servingMode=serving_mode,
|
||||
chatRequest=OCIChatRequestPayload(
|
||||
apiFormat=vendor.value,
|
||||
messages=adapt_messages_to_generic_oci_standard(
|
||||
list(messages_with_file_parts_as_image_urls(messages))
|
||||
),
|
||||
messages=adapt_messages_to_generic_oci_standard(normalized_messages),
|
||||
**self._get_optional_params(vendor, optional_params, model),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -38,8 +38,13 @@ from itertools import accumulate, chain, groupby, repeat
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
|
||||
|
||||
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
from typing_extensions import (
|
||||
ReadOnly,
|
||||
TypedDict,
|
||||
TypeIs, # noqa: TID251 # TypeIs reaches typing only on 3.13
|
||||
)
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
|
|
@ -99,6 +104,7 @@ from litellm.types.llms.openai import (
|
|||
OutputTextDeltaEvent,
|
||||
OutputTextDoneEvent,
|
||||
ResponseAPIUsage,
|
||||
ResponseCompactionItem,
|
||||
ResponseCompletedEvent,
|
||||
ResponsesAPIOptionalRequestParams,
|
||||
ResponsesAPIResponse,
|
||||
|
|
@ -277,6 +283,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
|
|||
)
|
||||
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
|
||||
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
|
||||
_TEXT_DELTA_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.refusal.delta"})
|
||||
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
|
||||
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
|
||||
|
|
@ -537,6 +544,14 @@ def _scanned_text_field(item: Mapping[str, object]) -> str:
|
|||
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
|
||||
|
||||
|
||||
def _content_part_text_field(part: object) -> str:
|
||||
return "refusal" if stream_item_field(part, "type") == "refusal" else "text"
|
||||
|
||||
|
||||
def _has_writable_field(part: object, field: str) -> bool:
|
||||
return isinstance(part, dict) or hasattr(part, field)
|
||||
|
||||
|
||||
def _tool_output_part_text(part: object) -> str | None:
|
||||
if not isinstance(part, dict):
|
||||
return None
|
||||
|
|
@ -702,6 +717,15 @@ def _next_stream_sequence_number(responses_so_far: Sequence[object] | None) -> i
|
|||
return next((n + 1 for n in sequence_numbers if isinstance(n, int)), 0)
|
||||
|
||||
|
||||
def _is_responses_input(
|
||||
input_data: object,
|
||||
) -> TypeIs["str | ResponseInputParam"]: # guard-ok: str, or a list of Mapping items
|
||||
if isinstance(input_data, str):
|
||||
return True
|
||||
items: Final = cast(list[object], input_data) # cast-ok: isinstance narrows to list[Unknown] only
|
||||
return isinstance(input_data, list) and all(isinstance(item, (Mapping, ResponseCompactionItem)) for item in items)
|
||||
|
||||
|
||||
class OpenAIResponsesHandler(BaseTranslation):
|
||||
"""
|
||||
Handler for processing OpenAI Responses API with guardrails.
|
||||
|
|
@ -723,8 +747,8 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
Transforms `input` (string or ResponseInputParam) and optional
|
||||
`instructions` into chat completion messages.
|
||||
"""
|
||||
input_data: Final = data.get("input")
|
||||
if input_data is None:
|
||||
input_data: Final[object] = data.get("input")
|
||||
if not _is_responses_input(input_data):
|
||||
return None
|
||||
messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
|
||||
input=input_data,
|
||||
|
|
@ -993,7 +1017,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
for content_idx, content_item in enumerate(content):
|
||||
if isinstance(content_item, dict):
|
||||
# Extract text
|
||||
text_str = content_item.get("text", None)
|
||||
text_str = content_item.get(_content_part_text_field(content_item), None)
|
||||
if text_str is not None:
|
||||
texts_to_check.append(text_str)
|
||||
task_mappings.append((msg_idx, int(content_idx)))
|
||||
|
|
@ -1042,8 +1066,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
elif isinstance(content, list):
|
||||
# Replace specific text item in list content
|
||||
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
|
||||
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
|
||||
self._write_content_part_text(content[content_idx_optional], guardrail_response)
|
||||
|
||||
async def process_output_response(
|
||||
self,
|
||||
|
|
@ -1380,7 +1403,8 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if isinstance(output_index, int) and isinstance(content_index, int)
|
||||
)
|
||||
if len(placeable_positions) != len(scanned_positions) or any(
|
||||
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES for event in scanned_events
|
||||
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES | _TEXT_DELTA_EVENT_TYPES
|
||||
for event in scanned_events
|
||||
):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
|
|
@ -1401,6 +1425,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
else:
|
||||
setattr(event, field, value)
|
||||
|
||||
@staticmethod
|
||||
def _write_content_part_text(part: object, text: str) -> None:
|
||||
field: Final = _content_part_text_field(part)
|
||||
if _has_writable_field(part, field):
|
||||
OpenAIResponsesHandler._write_event_field(part, field, text)
|
||||
|
||||
@staticmethod
|
||||
def _write_compaction_summary(item: object, summary: str) -> None:
|
||||
encrypted_content: Final = stream_item_field(item, "encrypted_content")
|
||||
|
|
@ -1416,11 +1446,12 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
) -> None:
|
||||
"""Sync pre-completion stream events with the rewritten completed
|
||||
response, keyed by ``(output_index, content_index)``: the first
|
||||
``output_text.delta`` for a rewritten item carries the full rewritten
|
||||
text and the rest are blanked, while ``output_text.done``,
|
||||
``content_part.done``, and ``output_item.done`` events carry the full
|
||||
rewritten text, so every event a client may read agrees with the
|
||||
rewritten ``response.completed`` payload."""
|
||||
``output_text.delta`` or ``refusal.delta`` for a rewritten item carries
|
||||
the full rewritten text and the rest are blanked, while
|
||||
``output_text.done``, ``refusal.done``, ``content_part.done``, and
|
||||
``output_item.done`` events carry the full rewritten text, so every
|
||||
event a client may read agrees with the rewritten ``response.completed``
|
||||
payload."""
|
||||
delta_replacements: Final = MappingProxyType(
|
||||
{position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()}
|
||||
)
|
||||
|
|
@ -1436,14 +1467,14 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
if not isinstance(output_index, int) or not isinstance(content_index, int):
|
||||
continue
|
||||
position = (output_index, content_index)
|
||||
if event_type == "response.output_text.delta" and position in delta_replacements:
|
||||
if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements:
|
||||
self._write_event_field(event, "delta", next(delta_replacements[position]))
|
||||
elif event_type == "response.output_text.done" and position in rewrites_by_position:
|
||||
self._write_event_field(event, "text", rewrites_by_position[position])
|
||||
elif event_type == "response.refusal.done" and position in rewrites_by_position:
|
||||
self._write_event_field(event, "refusal", rewrites_by_position[position])
|
||||
elif event_type == "response.content_part.done" and position in rewrites_by_position:
|
||||
part = event.get("part")
|
||||
if isinstance(part, dict) or hasattr(part, "text"):
|
||||
self._write_event_field(part, "text", rewrites_by_position[position])
|
||||
self._write_content_part_text(event.get("part"), rewrites_by_position[position])
|
||||
|
||||
@staticmethod
|
||||
def _sync_output_item_done_event(
|
||||
|
|
@ -1459,7 +1490,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
for (item_idx, content_idx), rewritten in rewrites_by_position.items():
|
||||
if item_idx != output_index or not 0 <= content_idx < len(content):
|
||||
continue
|
||||
OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten)
|
||||
OpenAIResponsesHandler._write_content_part_text(content[content_idx], rewritten)
|
||||
|
||||
def _deliver_ended_stream_tool_call_rewrites(
|
||||
self,
|
||||
|
|
@ -1814,7 +1845,7 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
return
|
||||
|
||||
# Handle both GenericResponseOutputItem and dict
|
||||
content: list[OutputText] | list[dict] | None = None
|
||||
content: list[OutputText | ResponseOutputRefusal] | list[dict[str, object]] | None = None
|
||||
if isinstance(output_item, BaseModel):
|
||||
try:
|
||||
output_item_dump: Final = output_item.model_dump()
|
||||
|
|
@ -1837,17 +1868,9 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
|
||||
verbose_proxy_logger.debug("OpenAI Responses API: Processing output item: %s", output_item)
|
||||
|
||||
# Iterate through content items (list of OutputText objects)
|
||||
for content_idx, content_item in enumerate(content):
|
||||
# Handle both OutputText objects and dicts
|
||||
if isinstance(content_item, OutputText):
|
||||
text_content = content_item.text
|
||||
elif isinstance(content_item, dict):
|
||||
text_content = content_item.get("text")
|
||||
else:
|
||||
continue
|
||||
|
||||
if text_content:
|
||||
text_content = stream_item_field(content_item, _content_part_text_field(content_item))
|
||||
if isinstance(text_content, str) and text_content:
|
||||
texts_to_check.append(text_content)
|
||||
task_mappings.append((output_idx, int(content_idx)))
|
||||
|
||||
|
|
@ -1887,33 +1910,24 @@ class OpenAIResponsesHandler(BaseTranslation):
|
|||
# Handle both GenericResponseOutputItem, BaseModel, and dict
|
||||
if isinstance(output_item, GenericResponseOutputItem):
|
||||
if output_item.content and content_idx < len(output_item.content):
|
||||
content_item = output_item.content[content_idx]
|
||||
if isinstance(content_item, OutputText):
|
||||
content_item.text = guardrail_response
|
||||
elif isinstance(content_item, dict):
|
||||
content_item["text"] = guardrail_response
|
||||
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
|
||||
elif isinstance(output_item, BaseModel):
|
||||
# Handle other Pydantic models by converting to GenericResponseOutputItem
|
||||
try:
|
||||
generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())
|
||||
if generic_item.content and content_idx < len(generic_item.content):
|
||||
content_item = generic_item.content[content_idx]
|
||||
if isinstance(content_item, OutputText):
|
||||
content_item.text = guardrail_response
|
||||
# Update the original response output
|
||||
if hasattr(output_item, "content") and output_item.content:
|
||||
original_content = output_item.content[content_idx]
|
||||
if hasattr(original_content, "text"):
|
||||
original_content.text = guardrail_response
|
||||
if (
|
||||
generic_item.content
|
||||
and content_idx < len(generic_item.content)
|
||||
and hasattr(output_item, "content")
|
||||
and output_item.content
|
||||
):
|
||||
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
|
||||
except Exception:
|
||||
pass
|
||||
elif isinstance(output_item, dict):
|
||||
content = output_item.get("content", [])
|
||||
if content and content_idx < len(content):
|
||||
if isinstance(content[content_idx], dict):
|
||||
content[content_idx]["text"] = guardrail_response
|
||||
elif hasattr(content[content_idx], "text"):
|
||||
content[content_idx].text = guardrail_response
|
||||
self._write_content_part_text(content[content_idx], guardrail_response)
|
||||
|
||||
def build_block_sse_chunks(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -129,6 +129,17 @@ def _native_vertex_row_usage(
|
|||
return calculate_usage(completion_response)
|
||||
|
||||
|
||||
def native_vertex_batch_row_model(row: Mapping[str, object], model_name: str | None) -> str | None:
|
||||
response: Final = as_json_object(row.get("response"))
|
||||
model_version: Final = (
|
||||
response.get("model" if is_vertex_anthropic_batch_output_response(response) else "modelVersion")
|
||||
if response is not None
|
||||
else None
|
||||
)
|
||||
deployment_model: Final = model_name if model_name and "*" not in model_name else None
|
||||
return deployment_model or (model_version if isinstance(model_version, str) else model_name)
|
||||
|
||||
|
||||
def native_vertex_batch_row_stats(
|
||||
row: Mapping[str, object],
|
||||
model_name: str | None,
|
||||
|
|
@ -156,11 +167,7 @@ def native_vertex_batch_row_stats(
|
|||
if usage is None:
|
||||
return None
|
||||
total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
|
||||
model_version: Final = response_body.get(
|
||||
"model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
|
||||
)
|
||||
deployment_model: Final = model_name if model_name and "*" not in model_name else None
|
||||
model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
|
||||
model: Final = native_vertex_batch_row_model(row, model_name)
|
||||
if model is None:
|
||||
verbose_logger.warning(
|
||||
"vertex_ai batch output row could not be costed, so it is billed at $0 and the rest of the batch "
|
||||
|
|
|
|||
|
|
@ -31,9 +31,8 @@ from litellm.exceptions import UnsupportedParamsError
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
encode_tool_call_id_with_signature,
|
||||
)
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -2091,6 +2090,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
parts: list[HttpxPartType],
|
||||
cumulative_tool_call_idx: int,
|
||||
is_function_call: bool | None,
|
||||
thought_signature_in_tool_call_id: bool = True,
|
||||
) -> tuple[
|
||||
ChatCompletionToolCallFunctionChunk | None,
|
||||
list[ChatCompletionToolCallChunk] | None,
|
||||
|
|
@ -2129,9 +2129,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# back on the matching `functionResponse`.
|
||||
if gemini_call_id:
|
||||
_tool_response_chunk["id"] = gemini_call_id
|
||||
# Embed thought signature in ID for OpenAI client compatibility
|
||||
if thought_signature:
|
||||
_tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature}
|
||||
if thought_signature and thought_signature_in_tool_call_id:
|
||||
_tool_response_chunk["id"] = encode_tool_call_id_with_signature(
|
||||
_tool_response_chunk["id"] or "", thought_signature
|
||||
)
|
||||
|
|
@ -2793,6 +2793,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
model_response: Union[ModelResponse, "ModelResponseStream"],
|
||||
standard_optional_params: dict,
|
||||
cumulative_tool_call_index: int = 0,
|
||||
thought_signature_in_tool_call_id: bool = True,
|
||||
) -> tuple[list[dict], list[dict], list, list, int]:
|
||||
"""
|
||||
Helper method to process candidates and extract metadata
|
||||
|
|
@ -2895,6 +2896,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
parts=candidate["content"]["parts"],
|
||||
cumulative_tool_call_idx=cumulative_tool_call_index,
|
||||
is_function_call=is_function_call(standard_optional_params),
|
||||
thought_signature_in_tool_call_id=thought_signature_in_tool_call_id,
|
||||
)
|
||||
|
||||
if "logprobsResult" in candidate:
|
||||
|
|
@ -2978,6 +2980,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
model_response: Union[ModelResponse, "ModelResponseStream"],
|
||||
standard_optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
cumulative_tool_call_index: int = 0,
|
||||
thought_signature_in_tool_call_id: bool = True,
|
||||
) -> tuple[ # mutable-ok: mirrors override contract
|
||||
list[dict[str, object]],
|
||||
list[dict[str, object]],
|
||||
|
|
@ -2986,7 +2989,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
int,
|
||||
]:
|
||||
return cls._process_candidates(
|
||||
_candidates, model_response, standard_optional_params, cumulative_tool_call_index
|
||||
_candidates,
|
||||
model_response,
|
||||
standard_optional_params,
|
||||
cumulative_tool_call_index,
|
||||
thought_signature_in_tool_call_id,
|
||||
)
|
||||
|
||||
def transform_response(
|
||||
|
|
@ -3069,7 +3076,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
safety_ratings,
|
||||
citation_metadata,
|
||||
_, # cumulative_tool_call_index not needed in non-streaming
|
||||
) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params)
|
||||
) = VertexGeminiConfig._process_candidates(
|
||||
_candidates,
|
||||
model_response,
|
||||
logging_obj.optional_params,
|
||||
thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id(
|
||||
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
|
||||
),
|
||||
)
|
||||
|
||||
usage: Final = VertexGeminiConfig._calculate_usage(
|
||||
completion_response=completion_response,
|
||||
|
|
@ -3905,6 +3919,9 @@ class ModelResponseIterator:
|
|||
self.logging_obj = logging_obj
|
||||
self.response_headers = response_headers or {}
|
||||
self.is_function_call = check_is_function_call(logging_obj)
|
||||
self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id(
|
||||
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
|
||||
)
|
||||
self.cumulative_tool_call_index: int = 0
|
||||
self.has_seen_tool_calls: bool = False
|
||||
self._resume = resume
|
||||
|
|
@ -3969,6 +3986,7 @@ class ModelResponseIterator:
|
|||
model_response,
|
||||
self.logging_obj.optional_params,
|
||||
cumulative_tool_call_index=self.cumulative_tool_call_index,
|
||||
thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id,
|
||||
)
|
||||
|
||||
# Track whether tool_calls have been seen across streaming chunks.
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin
|
|||
from concurrent import futures
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
|
||||
|
|
@ -82,8 +83,8 @@ from litellm.litellm_core_utils.dd_tracing import tracer
|
|||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
AWS_CREDENTIAL_KWARGS_KEYS,
|
||||
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
|
||||
OPTIONAL_KWARGS_KEYS,
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
SUPPLEMENTAL_KWARGS_KEYS,
|
||||
InvalidControlOption,
|
||||
parse_control_options,
|
||||
with_control_options,
|
||||
|
|
@ -157,7 +158,7 @@ from litellm.types.completion import (
|
|||
)
|
||||
from litellm.types.litellm_params import ControlOptions, RetryStrategy
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly
|
||||
from litellm.types.utils import (
|
||||
AgenticLoopParams,
|
||||
BaseUrlRoutingHint,
|
||||
|
|
@ -399,6 +400,49 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
|
|||
return assert_never(control)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SupplementalProviderParams:
|
||||
params: GenericLiteLLMParams
|
||||
dropped_keys: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _InvalidSupplementalParams:
|
||||
message: str
|
||||
param: str
|
||||
|
||||
|
||||
def _parse_supplemental_provider_params(
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _SupplementalProviderParams | _InvalidSupplementalParams:
|
||||
supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs}
|
||||
try:
|
||||
return _SupplementalProviderParams(params=validate_litellm_params_strictly(GenericLiteLLMParams, supplemental))
|
||||
except ValidationError as error:
|
||||
invalid: Final = tuple(detail for detail in error.errors() if detail["loc"])
|
||||
invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid)
|
||||
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
|
||||
return _SupplementalProviderParams(
|
||||
params=GenericLiteLLMParams.model_validate(
|
||||
{key: value for key, value in supplemental.items() if key not in invalid_keys}
|
||||
),
|
||||
dropped_keys=invalid_keys,
|
||||
)
|
||||
return _InvalidSupplementalParams(
|
||||
message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid),
|
||||
param=", ".join(sorted(invalid_keys)),
|
||||
)
|
||||
|
||||
|
||||
def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams:
|
||||
parsed: Final = _parse_supplemental_provider_params(kwargs)
|
||||
if isinstance(parsed, _InvalidSupplementalParams):
|
||||
raise litellm.BadRequestError(
|
||||
message=parsed.message, model=model, llm_provider=None, body={"param": parsed.param}
|
||||
)
|
||||
return parsed
|
||||
|
||||
|
||||
PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
|
||||
PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
|
||||
|
||||
|
|
@ -830,21 +874,17 @@ async def acompletion(
|
|||
"enable_json_schema_validation": enable_json_schema_validation,
|
||||
}
|
||||
if custom_llm_provider is None:
|
||||
_supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects
|
||||
"dict[str, object]",
|
||||
{
|
||||
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
|
||||
"extra_headers": extra_headers,
|
||||
},
|
||||
)
|
||||
request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs
|
||||
_, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=cast( # cast-ok: api_base arrives as a str in the request kwargs
|
||||
"str | None", kwargs.get("api_base")
|
||||
"str | None", request_kwargs.get("api_base")
|
||||
)
|
||||
or base_url,
|
||||
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
|
||||
litellm_params=_supplemental_provider_params(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
|
||||
).params,
|
||||
)
|
||||
|
||||
if fallbacks is not None:
|
||||
|
|
@ -1197,6 +1237,7 @@ def mock_completion(
|
|||
custom_llm_provider="openai",
|
||||
logging_obj=logging,
|
||||
count_prompt_tokens=count_prompt_tokens,
|
||||
is_mock=True,
|
||||
)
|
||||
return CustomStreamWrapper(
|
||||
completion_stream=mock_completion_streaming_obj(
|
||||
|
|
@ -1206,6 +1247,7 @@ def mock_completion(
|
|||
custom_llm_provider="openai",
|
||||
logging_obj=logging,
|
||||
count_prompt_tokens=count_prompt_tokens,
|
||||
is_mock=True,
|
||||
)
|
||||
if isinstance(mock_response, litellm.MockException):
|
||||
raise mock_response
|
||||
|
|
@ -4382,6 +4424,7 @@ _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
|
|||
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
|
||||
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
|
||||
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
|
||||
|
|
@ -5625,6 +5668,11 @@ def completion(
|
|||
|
||||
######### unpacking kwargs #####################
|
||||
args: Final = _locals_snapshot(locals())
|
||||
supplemental_params: Final = _supplemental_provider_params(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
|
||||
)
|
||||
for dropped_key in supplemental_params.dropped_keys:
|
||||
kwargs.pop(dropped_key)
|
||||
|
||||
# Set by the responses->completion fallback so completion() does not bridge
|
||||
# back to the Responses API: that round-trip mutually recurses forever for a
|
||||
|
|
@ -5850,21 +5898,14 @@ def completion(
|
|||
if deployment_id is not None: # azure llms
|
||||
model = deployment_id
|
||||
custom_llm_provider = "azure"
|
||||
_supplemental_provider_params: Final = {
|
||||
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
|
||||
"extra_headers": extra_headers,
|
||||
}
|
||||
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
|
||||
requested_model: Final = model
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
litellm_params=(
|
||||
GenericLiteLLMParams.model_validate(_supplemental_provider_params)
|
||||
if _supplemental_provider_params
|
||||
else None
|
||||
),
|
||||
litellm_params=supplemental_params.params,
|
||||
)
|
||||
_record_agentic_loop_params(
|
||||
logging_obj=logging,
|
||||
|
|
@ -5884,7 +5925,6 @@ def completion(
|
|||
api_base=api_base,
|
||||
deployment_model_info=deployment_model_info,
|
||||
)
|
||||
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
|
||||
drop_unsupported_reasoning: Final = (
|
||||
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
|
||||
)
|
||||
|
|
@ -6220,6 +6260,7 @@ def completion(
|
|||
"github_copilot_user_session",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
"fireworks_forward_user_id",
|
||||
"thought_signature_in_tool_call_id",
|
||||
"responses_api_version",
|
||||
"allowed_openai_params",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -81372,5 +81372,77 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"gpt-6-astra-law": {
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
"input_cost_per_token": 1e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-05,
|
||||
"reasoning_contexts": [
|
||||
"auto",
|
||||
"current_turn",
|
||||
"all_turns"
|
||||
],
|
||||
"reasoning_modes": [
|
||||
"standard",
|
||||
"pro"
|
||||
],
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_tool_search": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"chatgpt/gpt-6-astra-law": {
|
||||
"litellm_provider": "chatgpt",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "responses",
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24268,7 +24268,14 @@
|
|||
"properties": {
|
||||
"content": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/OutputText"
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/OutputText"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ResponseOutputRefusal"
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": "Content",
|
||||
"type": "array"
|
||||
|
|
|
|||
|
|
@ -38,12 +38,14 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
|||
validate_no_callback_env_reference,
|
||||
)
|
||||
from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled
|
||||
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
|
||||
from litellm.types.agents import (
|
||||
AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.types.agents import (
|
||||
AgentResponse as AgentResponse, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.types.decisions import OpenAIDecisionResponse
|
||||
from litellm.types.integrations.compression_interception import (
|
||||
CompressionSavingsMetadata,
|
||||
)
|
||||
|
|
@ -2470,6 +2472,22 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
Documents all the fields supported by `general_settings` in config.yaml
|
||||
"""
|
||||
|
||||
accounting_capture: UsageCaptureSettings | None = Field(
|
||||
None,
|
||||
description="Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes.",
|
||||
)
|
||||
accounting_sync_ack: bool = Field(
|
||||
False,
|
||||
description="Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. "
|
||||
"Other requests remain supported with asynchronous usage capture.",
|
||||
)
|
||||
accounting_sync_ack_timeout_seconds: float = Field(
|
||||
20.0, gt=0, description="Maximum response wait for usage commit, including the batch flush interval"
|
||||
)
|
||||
accounting_sync_ack_failure_policy: Literal["fail_closed", "log"] = Field(
|
||||
"fail_closed", description="Return 503 or log the persistence failure after the response wait expires"
|
||||
)
|
||||
|
||||
request_tag_mappings: Sequence[RequestTagMapping] | None = Field(
|
||||
None,
|
||||
description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.",
|
||||
|
|
@ -4584,6 +4602,7 @@ PassThroughEndpointLoggingResultValues = (
|
|||
| StandardPassThroughResponseObject
|
||||
| ResponsesAPIResponse
|
||||
| TranscriptionResponse
|
||||
| OpenAIDecisionResponse
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping
|
||||
from copy import deepcopy
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Protocol
|
||||
|
|
@ -19,7 +19,8 @@ from urllib.parse import urlparse
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from fastapi.routing import APIRoute
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -27,7 +28,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
|||
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
|
||||
from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
|
||||
from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.a2a.version_convert import (
|
||||
A2AVersion,
|
||||
normalize_agent_card,
|
||||
|
|
@ -42,6 +43,7 @@ from litellm.proxy.agent_endpoints.databricks_oauth import (
|
|||
)
|
||||
from litellm.proxy.agent_endpoints.utils import merge_agent_headers
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING,
|
||||
coerce_keepalive_interval,
|
||||
|
|
@ -172,13 +174,16 @@ def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str,
|
|||
async def _resolve_backend_auth(
|
||||
litellm_params: dict[str, object],
|
||||
custom_llm_provider: object,
|
||||
agent_extra_headers: Mapping[str, str] | None = None,
|
||||
) -> AgentBackendAuth:
|
||||
databricks_auth: Final = await resolve_databricks_backend_auth(litellm_params)
|
||||
if has_databricks_oauth(litellm_params):
|
||||
return databricks_auth
|
||||
return AgentBackendAuth(
|
||||
litellm_params=databricks_auth.litellm_params,
|
||||
header=await resolve_a2a_hop_auth_header(litellm_params, custom_llm_provider),
|
||||
header=await resolve_a2a_hop_auth_header(
|
||||
litellm_params, custom_llm_provider, agent_extra_headers=agent_extra_headers
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -218,6 +223,57 @@ def _jsonrpc_error(
|
|||
)
|
||||
|
||||
|
||||
class _JsonRpcRequestId(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str | int | None = None
|
||||
|
||||
|
||||
async def _jsonrpc_request_id(request: Request) -> str | int | None:
|
||||
try:
|
||||
return _JsonRpcRequestId.model_validate_json(await request.body()).id
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
async def _jsonrpc_refusal(request: Request, refusal: ProxyException | HTTPException) -> JSONResponse:
|
||||
from litellm.proxy.proxy_server import (
|
||||
_close_dangling_otel_server_span, # pyright: ignore[reportPrivateUsage] # proxy_server keeps the span-close helper private; a route that answers an error itself must stamp the OTel server span like the global ProxyException handler does
|
||||
)
|
||||
|
||||
error: Final = refusal if isinstance(refusal, ProxyException) else status_typed_proxy_exception(refusal)
|
||||
_close_dangling_otel_server_span(request, int(error.code), exc=refusal)
|
||||
return JSONResponse(
|
||||
content={
|
||||
"jsonrpc": "2.0",
|
||||
"id": await _jsonrpc_request_id(request),
|
||||
"error": {
|
||||
"code": -32000,
|
||||
"message": error.message,
|
||||
"data": {"type": error.type, "param": error.param, "code": error.code},
|
||||
},
|
||||
},
|
||||
status_code=int(error.code),
|
||||
headers=error.headers if isinstance(refusal, ProxyException) else refusal.headers,
|
||||
)
|
||||
|
||||
|
||||
class _JsonRpcRefusalRoute(APIRoute):
|
||||
def get_route_handler(self) -> Callable[[Request], Coroutine[object, object, Response]]:
|
||||
handler: Final = super().get_route_handler()
|
||||
|
||||
async def answer_refusals_as_jsonrpc(request: Request) -> Response:
|
||||
try:
|
||||
return await handler(request)
|
||||
except (ProxyException, HTTPException) as refusal:
|
||||
return await _jsonrpc_refusal(request, refusal)
|
||||
|
||||
return answer_refusals_as_jsonrpc
|
||||
|
||||
|
||||
jsonrpc_router: Final = APIRouter(route_class=_JsonRpcRefusalRoute)
|
||||
|
||||
|
||||
async def _get_agent(agent_id: str) -> "AgentResponse | None":
|
||||
"""Look up an agent by ID or name. Returns None if not found."""
|
||||
from litellm.proxy.common_utils.registry_read_through import (
|
||||
|
|
@ -646,17 +702,17 @@ async def get_agent_card(
|
|||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post(
|
||||
@jsonrpc_router.post(
|
||||
"/a2a/{agent_id}",
|
||||
tags=["[beta] A2A Agents"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
@router.post(
|
||||
@jsonrpc_router.post(
|
||||
"/a2a/{agent_id}/message/send",
|
||||
tags=["[beta] A2A Agents"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
@router.post(
|
||||
@jsonrpc_router.post(
|
||||
"/v1/a2a/{agent_id}/message/send",
|
||||
tags=["[beta] A2A Agents"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
|
|
@ -834,14 +890,15 @@ async def invoke_agent_a2a(
|
|||
if header_name:
|
||||
dynamic_headers[header_name] = val
|
||||
|
||||
backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider)
|
||||
merged_agent_headers: Final = merge_agent_headers(
|
||||
dynamic_headers=dynamic_headers or None,
|
||||
static_headers=static_headers or None,
|
||||
)
|
||||
backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider, merged_agent_headers)
|
||||
agent_extra_headers: Final = _forwarding_headers(
|
||||
caller_identity=caller_identity,
|
||||
request_data=data,
|
||||
agent_extra_headers=merge_agent_headers(
|
||||
dynamic_headers=dynamic_headers or None,
|
||||
static_headers=static_headers or None,
|
||||
),
|
||||
agent_extra_headers=merged_agent_headers,
|
||||
backend_auth_header=backend_auth.header,
|
||||
)
|
||||
forwarded_litellm_params: Final = backend_auth.litellm_params
|
||||
|
|
@ -1051,3 +1108,6 @@ async def invoke_agent_a2a(
|
|||
if isinstance(e, litellm.BadRequestError):
|
||||
return _jsonrpc_error(body.get("id"), -32602, e.message, 400)
|
||||
return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500)
|
||||
|
||||
|
||||
router.include_router(jsonrpc_router)
|
||||
|
|
|
|||
|
|
@ -1086,13 +1086,31 @@ def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] |
|
|||
|
||||
|
||||
def route_skips_budget_checks(route: str) -> bool:
|
||||
return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
|
||||
route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)
|
||||
if route in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES:
|
||||
return False
|
||||
if route in MODEL_DISCOVERY_ROUTES:
|
||||
return True
|
||||
if RouteChecks.is_llm_api_route(route=route):
|
||||
return False
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( # noqa: PLC0415 # pass-through endpoints import the auth stack
|
||||
InitPassThroughEndpointHelpers,
|
||||
)
|
||||
|
||||
return not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route)
|
||||
|
||||
def request_skips_budget_checks(route: str, model: str | list[str] | None, llm_router: Router | None) -> bool:
|
||||
return route_skips_budget_checks(route=route) or is_model_cost_zero(model=model, llm_router=llm_router)
|
||||
|
||||
def request_skips_budget_checks(
|
||||
route: str,
|
||||
model: str | list[str] | None,
|
||||
llm_router: Router | None,
|
||||
*,
|
||||
request_body: Mapping[str, object] | None = None,
|
||||
) -> bool:
|
||||
if route_skips_budget_checks(route=route):
|
||||
return True
|
||||
if request_body is not None and is_mcp_discovery_request(route=route, request_body=request_body):
|
||||
return True
|
||||
return is_model_cost_zero(model=model, llm_router=llm_router)
|
||||
|
||||
|
||||
async def common_checks(
|
||||
|
|
@ -1142,10 +1160,11 @@ async def common_checks(
|
|||
team_id=valid_token.team_id if valid_token is not None else None,
|
||||
)
|
||||
|
||||
skip_all_budget_checks: Final = (
|
||||
skip_budget_checks
|
||||
or route_skips_budget_checks(route=route)
|
||||
or is_mcp_discovery_request(route=route, request_body=_typed_request_body(request_body))
|
||||
skip_all_budget_checks: Final = skip_budget_checks or request_skips_budget_checks(
|
||||
route=route,
|
||||
model=_model,
|
||||
llm_router=llm_router,
|
||||
request_body=_typed_request_body(request_body),
|
||||
)
|
||||
|
||||
membership_user_id: Final = (
|
||||
|
|
|
|||
|
|
@ -80,7 +80,6 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
get_team_object,
|
||||
get_user_object,
|
||||
get_user_role,
|
||||
is_mcp_discovery_request,
|
||||
is_model_cost_zero,
|
||||
is_user_proxy_admin,
|
||||
is_valid_fallback_model,
|
||||
|
|
@ -88,6 +87,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
key_access_group_grants_model,
|
||||
key_model_aliases_for_auth_check,
|
||||
model_max_budget_of,
|
||||
request_skips_budget_checks,
|
||||
resolve_and_validate_end_user_id,
|
||||
resolve_default_end_user_budget,
|
||||
route_skips_budget_checks,
|
||||
|
|
@ -2497,15 +2497,9 @@ async def user_api_key_auth_builder(
|
|||
llm_router=llm_router,
|
||||
team_id=valid_token.team_id,
|
||||
)
|
||||
skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
|
||||
if not skip_budget_checks and model is not None and llm_router is not None:
|
||||
from litellm.proxy.auth.auth_checks import is_model_cost_zero
|
||||
|
||||
skip_budget_checks = ( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
is_model_cost_zero(model=model, llm_router=llm_router)
|
||||
)
|
||||
if skip_budget_checks:
|
||||
verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
|
||||
skip_budget_checks = request_skips_budget_checks(
|
||||
route=route, model=model, llm_router=llm_router, request_body=request_data
|
||||
)
|
||||
|
||||
# Fetch project object for JWT path if project_id is set
|
||||
_jwt_project_obj = None
|
||||
|
|
@ -3019,15 +3013,9 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
llm_router=llm_router,
|
||||
team_id=valid_token.team_id,
|
||||
)
|
||||
skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
|
||||
if not skip_budget_checks and model is not None and llm_router is not None:
|
||||
from litellm.proxy.auth.auth_checks import is_model_cost_zero
|
||||
|
||||
skip_budget_checks = is_model_cost_zero( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
model=model, llm_router=llm_router
|
||||
)
|
||||
if skip_budget_checks:
|
||||
verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
|
||||
skip_budget_checks: Final = request_skips_budget_checks(
|
||||
route=route, model=model, llm_router=llm_router, request_body=request_data
|
||||
)
|
||||
|
||||
# Check 3. Check if user is in their team budget
|
||||
if not skip_budget_checks and valid_token.team_member_spend is not None:
|
||||
|
|
@ -4088,11 +4076,7 @@ def _should_skip_budget_checks(
|
|||
llm_router=llm_router,
|
||||
team_id=team_id,
|
||||
)
|
||||
if is_mcp_discovery_request(route=route, request_body=request_data):
|
||||
return True
|
||||
if model is not None and llm_router is not None:
|
||||
return is_model_cost_zero(model=model, llm_router=llm_router)
|
||||
return False
|
||||
return request_skips_budget_checks(route=route, model=model, llm_router=llm_router, request_body=request_data)
|
||||
|
||||
|
||||
def _resolve_request_principal(request: Request, valid_token: UserAPIKeyAuth) -> Principal:
|
||||
|
|
|
|||
|
|
@ -137,6 +137,7 @@ from litellm.proxy.route_llm_request import (
|
|||
route_request,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost, prepare_response_cost
|
||||
from litellm.proxy.spend_tracking.usage_capture import capture_proxy_response
|
||||
from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
|
||||
from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
|
||||
ProxyLogging,
|
||||
|
|
@ -3366,6 +3367,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
)
|
||||
),
|
||||
)
|
||||
await capture_proxy_response(
|
||||
logging_obj, response, background=_REQUEST_FIELDS.validate_python(self.data).get("background") is True
|
||||
)
|
||||
except (Exception, asyncio.CancelledError) as exc:
|
||||
_exception_raised = True
|
||||
if logging_obj.prepared_cost is not None:
|
||||
|
|
|
|||
|
|
@ -275,7 +275,7 @@ class AccountingLedger:
|
|||
)
|
||||
return tuple(
|
||||
UsageWrite(
|
||||
OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
|
||||
OutboundUsageEvent.model_validate_json(_json(_JSON.validate_python(row["payload"])["event"])),
|
||||
_json(row["payload"]),
|
||||
TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,18 +12,31 @@ from collections.abc import Mapping, Sequence
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
|
||||
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
anthropic_message_delta_fields,
|
||||
anthropic_sse_error_events,
|
||||
anthropic_sse_event_type,
|
||||
anthropic_sse_event_types,
|
||||
anthropic_sse_started_blocks,
|
||||
anthropic_sse_stopped_block_index,
|
||||
anthropic_sse_unfinished_blocks,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
has_unreadable_json_event as has_unreadable_json_event, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
|
||||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
|
||||
|
|
@ -101,6 +114,48 @@ def _rewritten_message_delta_fields(
|
|||
)
|
||||
|
||||
|
||||
_MESSAGE_END_EVENT_TYPES: Final = frozenset({"message_delta", "message_stop"})
|
||||
|
||||
|
||||
def _text_block_left_open(rewritten_chunks: Sequence[bytes], source_chunks: Sequence[object]) -> int | None:
|
||||
"""Index of the rewritten text block to leave open when the failed upstream never stopped its text.
|
||||
|
||||
Assembly keeps a cut-off text block and drops every other cut-off block, and the round trip
|
||||
merges all text into one block, so that block carries text the upstream left open. It stays
|
||||
open only when it is the last block, since a client must see each block stop before the next
|
||||
one starts.
|
||||
"""
|
||||
upstream_left_text_open: Final = any(
|
||||
block.get("type") == "text" for block in anthropic_sse_unfinished_blocks(source_chunks).values()
|
||||
)
|
||||
rewritten_blocks: Final = anthropic_sse_started_blocks(rewritten_chunks)
|
||||
if not upstream_left_text_open or not rewritten_blocks:
|
||||
return None
|
||||
last_index: Final = max(rewritten_blocks)
|
||||
return last_index if rewritten_blocks[last_index].get("type") == "text" else None
|
||||
|
||||
|
||||
def _is_empty_text(block: object) -> bool:
|
||||
fields: Final = as_str_mapping(block)
|
||||
return fields is not None and fields.get("type") == "text" and not fields.get("text")
|
||||
|
||||
|
||||
def _without_empty_text(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
|
||||
return {**response, "content": [block for block in response.get("content") or () if not _is_empty_text(block)]}
|
||||
|
||||
|
||||
def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[str]) -> bool:
|
||||
event: Final = chunk.decode()
|
||||
event_type: Final = anthropic_sse_event_type(event)
|
||||
if event_type in _MESSAGE_END_EVENT_TYPES:
|
||||
return event_type in end_events_sent
|
||||
return block_left_open is None or anthropic_sse_stopped_block_index(event) != block_left_open
|
||||
|
||||
|
||||
def _sse_chunks_of_events(events: Sequence[str]) -> tuple[bytes, ...]:
|
||||
return tuple(f"{event}\n\n".encode() for event in events)
|
||||
|
||||
|
||||
def anthropic_sse_chunks_from_response(
|
||||
assembled: ModelResponse, source_chunks: Sequence[object] = (), *, keep_explanations: bool = False
|
||||
) -> tuple[bytes, ...]:
|
||||
|
|
@ -110,7 +165,26 @@ def anthropic_sse_chunks_from_response(
|
|||
nothing the OpenAI round trip made up, with a verdict kept only for a tool call delivered with
|
||||
the input the classifier judged. Their free-text ``explanation`` was written about the text the
|
||||
guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``).
|
||||
A source stream that ended in an upstream ``error`` event ends with that event again, never
|
||||
with a ``message_delta`` or ``message_stop`` the upstream did not send, with its cut-off text
|
||||
block left open unless another block follows it, and with no finished text block when it read
|
||||
no text, since Claude Code does not retry a stream that shows a finished block. A response that
|
||||
cannot be re-emitted, such as one with a tool input that cannot be parsed, leaves a failed stream
|
||||
its error alone.
|
||||
"""
|
||||
error_events: Final = anthropic_sse_error_events(source_chunks)
|
||||
try:
|
||||
return _rebuilt_sse_chunks(assembled, source_chunks, error_events, keep_explanations)
|
||||
except Exception:
|
||||
if not error_events:
|
||||
raise
|
||||
verbose_proxy_logger.warning("Failed Anthropic stream could not be rebuilt, sending its upstream error alone")
|
||||
return _sse_chunks_of_events(error_events)
|
||||
|
||||
|
||||
def _rebuilt_sse_chunks(
|
||||
assembled: ModelResponse, source_chunks: Sequence[object], error_events: Sequence[str], keep_explanations: bool
|
||||
) -> tuple[bytes, ...]:
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import (
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
)
|
||||
|
|
@ -120,7 +194,7 @@ def anthropic_sse_chunks_from_response(
|
|||
|
||||
adapter: Final = LiteLLMAnthropicMessagesAdapter()
|
||||
anthropic_response: Final = adapter.translate_openai_response_to_anthropic(response=assembled)
|
||||
source_response: Final = assemble_anthropic_sse_stream(source_chunks)
|
||||
source_response: Final = assemble_anthropic_sse_stream(source_chunks, through_upstream_error=True)
|
||||
source_content: Final = (
|
||||
adapter.translate_openai_response_to_anthropic(response=source_response).get("content")
|
||||
if source_response is not None
|
||||
|
|
@ -129,11 +203,15 @@ def anthropic_sse_chunks_from_response(
|
|||
unchanged_tool_use_ids: Final = _unchanged_tool_use_ids(
|
||||
source_content or (), anthropic_response.get("content") or ()
|
||||
)
|
||||
return tuple(
|
||||
FakeAnthropicMessagesStreamIterator(
|
||||
response=anthropic_response,
|
||||
message_delta_fields=_rewritten_message_delta_fields(
|
||||
source_chunks, unchanged_tool_use_ids, keep_explanations
|
||||
),
|
||||
).chunks
|
||||
rewritten_chunks: Final = FakeAnthropicMessagesStreamIterator(
|
||||
response=_without_empty_text(anthropic_response) if error_events else anthropic_response,
|
||||
message_delta_fields=_rewritten_message_delta_fields(source_chunks, unchanged_tool_use_ids, keep_explanations),
|
||||
).chunks
|
||||
if not error_events:
|
||||
return tuple(rewritten_chunks)
|
||||
block_left_open: Final = _text_block_left_open(rewritten_chunks, source_chunks)
|
||||
end_events_sent: Final = _MESSAGE_END_EVENT_TYPES & anthropic_sse_event_types(source_chunks)
|
||||
return (
|
||||
*(chunk for chunk in rewritten_chunks if _sent_by_the_failed_upstream(chunk, block_left_open, end_events_sent)),
|
||||
*_sse_chunks_of_events(error_events),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
|
|||
from litellm.proxy.guardrails.anthropic_sse import (
|
||||
anthropic_sse_chunks_from_response,
|
||||
assemble_anthropic_sse_stream,
|
||||
has_unreadable_json_event,
|
||||
is_anthropic_sse_stream,
|
||||
model_response_text,
|
||||
)
|
||||
|
|
@ -117,7 +118,7 @@ _MAX_FIRST_SSE_FRAME_BYTES: Final = 64 * 1024
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SsePreface:
|
||||
"""Complete leading SSE frames with no ``data:`` line, relayed verbatim before the stream shape is decided."""
|
||||
"""Complete leading SSE frames that carry no data, relayed verbatim before the stream shape is decided."""
|
||||
|
||||
raw: bytes
|
||||
|
||||
|
|
@ -125,17 +126,27 @@ class _SsePreface:
|
|||
_SSE_FRAME_END: Final = re.compile(rb"\r\n\r\n|\n\n|\r\r")
|
||||
|
||||
|
||||
def _carries_data(frame: bytes) -> bool:
|
||||
return any(line.startswith(b"data:") and line[len(b"data:") :].strip() for line in frame.splitlines())
|
||||
|
||||
|
||||
def _split_sse_preface(complete_frames: bytes) -> tuple[bytes, bytes]:
|
||||
"""Split complete frames into ``(frames before the first data-bearing frame, that frame and everything after)``."""
|
||||
start = 0
|
||||
for end in _SSE_FRAME_END.finditer(complete_frames):
|
||||
frame = complete_frames[start : end.end()]
|
||||
if any(line.startswith(b"data:") for line in frame.splitlines()):
|
||||
if _carries_data(frame):
|
||||
return complete_frames[:start], complete_frames[start:]
|
||||
start = end.end()
|
||||
return complete_frames, b""
|
||||
|
||||
|
||||
def _holds_json_object_frame(frames: bytes) -> bool:
|
||||
return any(
|
||||
line.startswith(b"data:") and line[len(b"data:") :].lstrip().startswith(b"{") for line in frames.splitlines()
|
||||
)
|
||||
|
||||
|
||||
def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[ModelResponseStream]:
|
||||
"""Buffered chunks flushed unmasked when a mixed stream shape makes reconstruction impossible."""
|
||||
if not all_chunks:
|
||||
|
|
@ -150,12 +161,13 @@ def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[
|
|||
|
||||
async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGenerator[object, None]:
|
||||
"""
|
||||
Relay leading data-less SSE frames (comment keepalives, events without a
|
||||
``data:`` line) as they complete, and join raw ``bytes`` chunks until they
|
||||
hold one complete SSE event with a data line, so the stream shape is
|
||||
decided on a whole frame rather than a transport fragment. Everything
|
||||
after that first frame is forwarded untouched. The byte cap can only be
|
||||
reached by a single unterminated frame.
|
||||
Relay leading data-less SSE frames (comment keepalives, events with no
|
||||
data) as they complete, and join raw ``bytes`` chunks until they
|
||||
hold one complete SSE event with a JSON object on its data line, so the
|
||||
stream shape is decided on a whole frame rather than a transport fragment
|
||||
or a line that says nothing about the shape. Everything after that frame
|
||||
is forwarded untouched. The byte cap bounds a single unterminated frame
|
||||
and a run of frames with no JSON object.
|
||||
"""
|
||||
pending = b""
|
||||
try:
|
||||
|
|
@ -169,7 +181,7 @@ async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGener
|
|||
if preface:
|
||||
yield _SsePreface(preface)
|
||||
pending = classifiable + tail
|
||||
if classifiable or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
|
||||
if _holds_json_object_frame(classifiable) or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
|
||||
break
|
||||
else:
|
||||
if pending:
|
||||
|
|
@ -1523,7 +1535,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
first_frame_is_anthropic = (
|
||||
not passthrough_due_to_unknown_stream_shape
|
||||
and not all_chunks
|
||||
and is_anthropic_sse_stream((chunk,))
|
||||
and self._is_anthropic_first_frame(chunk)
|
||||
)
|
||||
if not first_frame_is_anthropic:
|
||||
passthrough_due_to_unknown_stream_shape = (
|
||||
|
|
@ -1565,13 +1577,29 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
for chunk in all_chunks:
|
||||
yield chunk
|
||||
|
||||
def _is_anthropic_first_frame(self, chunk: bytes) -> bool:
|
||||
if is_anthropic_sse_stream((chunk,)):
|
||||
return True
|
||||
if has_unreadable_json_event((chunk,)):
|
||||
raise self._unreadable_stream_refusal()
|
||||
return False
|
||||
|
||||
def _unreadable_stream_refusal(self) -> GuardrailRaisedException:
|
||||
return GuardrailRaisedException(
|
||||
guardrail_name=self.guardrail_name,
|
||||
message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured",
|
||||
should_wrap_with_default_message=False,
|
||||
)
|
||||
|
||||
async def _mask_anthropic_sse_stream(
|
||||
self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict
|
||||
) -> tuple[object, ...]:
|
||||
rest_chunks: Final = [chunk async for chunk in rest]
|
||||
chunks: Final = (first_chunk, *rest_chunks)
|
||||
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True)
|
||||
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True)
|
||||
if assembled is None:
|
||||
if has_unreadable_json_event(chunks):
|
||||
raise self._unreadable_stream_refusal()
|
||||
verbose_proxy_logger.warning(
|
||||
"Presidio apply_to_output: raw SSE stream could not be assembled into a response. "
|
||||
"Output PII masking was skipped for this response."
|
||||
|
|
|
|||
|
|
@ -25,8 +25,10 @@ from litellm.proxy.common_utils.callback_utils import (
|
|||
)
|
||||
from litellm.proxy.guardrails.anthropic_sse import (
|
||||
anthropic_sse_chunks_from_response,
|
||||
anthropic_sse_error_events,
|
||||
assemble_anthropic_sse_stream,
|
||||
is_raw_sse_stream,
|
||||
is_sse_error_stream,
|
||||
)
|
||||
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
|
||||
|
|
@ -333,7 +335,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
parsed_arguments = arguments
|
||||
else:
|
||||
return None, "arguments must be a JSON object"
|
||||
except (json.JSONDecodeError, TypeError) as exc:
|
||||
except (json.JSONDecodeError, TypeError, RecursionError) as exc:
|
||||
verbose_proxy_logger.warning(
|
||||
"Tool Permission Guardrail: Failed to decode arguments for tool %s: %s",
|
||||
tool_call.function.name,
|
||||
|
|
@ -957,9 +959,11 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
yield chunk
|
||||
return
|
||||
|
||||
anthropic_response: Final = assemble_anthropic_sse_stream(all_chunks)
|
||||
anthropic_response: Final = assemble_anthropic_sse_stream(
|
||||
all_chunks, restore_identity=bool(anthropic_sse_error_events(all_chunks)), through_upstream_error=True
|
||||
)
|
||||
if anthropic_response is None:
|
||||
if is_raw_sse_stream(all_chunks):
|
||||
if is_raw_sse_stream(all_chunks) and not is_sse_error_stream(all_chunks):
|
||||
raise GuardrailRaisedException(
|
||||
guardrail_name=self.guardrail_name,
|
||||
message=(
|
||||
|
|
@ -971,16 +975,24 @@ class ToolPermissionGuardrail(CustomGuardrail):
|
|||
yield chunk
|
||||
return
|
||||
|
||||
anthropic_denials: Final = self._check_assembled_stream(anthropic_response)
|
||||
if not anthropic_denials:
|
||||
for chunk in all_chunks:
|
||||
yield chunk
|
||||
return
|
||||
|
||||
self._modify_response_with_permission_errors(anthropic_response, anthropic_denials)
|
||||
for sse_chunk in anthropic_sse_chunks_from_response(anthropic_response, all_chunks, keep_explanations=True):
|
||||
for sse_chunk in self._anthropic_stream_to_deliver(anthropic_response, all_chunks):
|
||||
yield sse_chunk
|
||||
|
||||
def _anthropic_stream_to_deliver(
|
||||
self, anthropic_response: ModelResponse, source_chunks: Sequence[object]
|
||||
) -> Sequence[object]:
|
||||
"""The frames the client gets: on a failed upstream only what the check read, never the upstream's own.
|
||||
|
||||
Assembly reads past an upstream error, and a client can read a frame of a broken stream that assembly
|
||||
does not, so forwarding those frames would pass a tool call nobody checked.
|
||||
"""
|
||||
denials: Final = self._check_assembled_stream(anthropic_response)
|
||||
if denials:
|
||||
self._modify_response_with_permission_errors(anthropic_response, denials)
|
||||
elif not anthropic_sse_error_events(source_chunks):
|
||||
return source_chunks
|
||||
return anthropic_sse_chunks_from_response(anthropic_response, source_chunks, keep_explanations=True)
|
||||
|
||||
def _check_assembled_stream(
|
||||
self, assembled: ModelResponse
|
||||
) -> tuple[tuple[ChatCompletionMessageToolCall, PermissionError], ...]:
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from litellm.proxy.auth.auth_utils import (
|
|||
)
|
||||
from litellm.proxy.auth.budget_throttle import throttled_limit
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
from litellm.types.utils import CallTypes, Usage
|
||||
|
||||
|
|
@ -41,10 +41,12 @@ def _precise_minute(now: datetime) -> str:
|
|||
return now.strftime("%Y-%m-%d-%H-%M")
|
||||
|
||||
|
||||
def _response_total_tokens(response_obj: object) -> int:
|
||||
if not isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)):
|
||||
return 0
|
||||
response_usage: Final = getattr(response_obj, "usage", None)
|
||||
def _response_total_tokens(kwargs: Mapping[str, object], response_obj: object) -> int:
|
||||
response_usage: Final = (
|
||||
getattr(response_obj, "usage", None)
|
||||
if isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse))
|
||||
else socket_session_usage(kwargs, response_obj)
|
||||
)
|
||||
return response_usage.total_tokens if isinstance(response_usage, Usage) else 0
|
||||
|
||||
|
||||
|
|
@ -556,7 +558,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
|
||||
precise_minute: Final = _precise_minute(self._clock())
|
||||
|
||||
total_tokens: int = _response_total_tokens(response_obj)
|
||||
total_tokens: Final = _response_total_tokens(
|
||||
kwargs, # pyright: ignore[reportUnknownArgumentType] # legacy callback kwargs are untyped
|
||||
response_obj,
|
||||
)
|
||||
|
||||
# ------------
|
||||
# Update usage - API Key
|
||||
|
|
@ -630,8 +635,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
# Update usage - User
|
||||
# ------------
|
||||
if user_api_key_user_id is not None:
|
||||
total_tokens = _response_total_tokens(response_obj)
|
||||
|
||||
request_count_api_key = f"{user_api_key_user_id}::{precise_minute}::request_count"
|
||||
|
||||
current = await self.internal_usage_cache.async_get_cache(
|
||||
|
|
@ -639,7 +642,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
litellm_parent_otel_span=litellm_parent_otel_span,
|
||||
) or {
|
||||
"current_requests": 1,
|
||||
"current_tpm": total_tokens,
|
||||
"current_tpm": 0,
|
||||
"current_rpm": 1,
|
||||
}
|
||||
|
||||
|
|
@ -656,8 +659,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
# Update usage - Team
|
||||
# ------------
|
||||
if user_api_key_team_id is not None:
|
||||
total_tokens = _response_total_tokens(response_obj)
|
||||
|
||||
request_count_api_key = f"{user_api_key_team_id}::{precise_minute}::request_count"
|
||||
|
||||
current = await self.internal_usage_cache.async_get_cache(
|
||||
|
|
@ -665,7 +666,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
litellm_parent_otel_span=litellm_parent_otel_span,
|
||||
) or {
|
||||
"current_requests": 1,
|
||||
"current_tpm": total_tokens,
|
||||
"current_tpm": 0,
|
||||
"current_rpm": 1,
|
||||
}
|
||||
|
||||
|
|
@ -682,8 +683,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
# Update usage - End User
|
||||
# ------------
|
||||
if user_api_key_end_user_id is not None:
|
||||
total_tokens = _response_total_tokens(response_obj)
|
||||
|
||||
request_count_api_key = f"{user_api_key_end_user_id}::{precise_minute}::request_count"
|
||||
|
||||
current = await self.internal_usage_cache.async_get_cache(
|
||||
|
|
@ -691,7 +690,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
|||
litellm_parent_otel_span=litellm_parent_otel_span,
|
||||
) or {
|
||||
"current_requests": 1,
|
||||
"current_tpm": total_tokens,
|
||||
"current_tpm": 0,
|
||||
"current_rpm": 1,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,8 +46,10 @@ from litellm.constants import (
|
|||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
|
||||
)
|
||||
from litellm.cost_calculator import get_usage_object
|
||||
from litellm.exceptions import ServiceUnavailableError
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
ensure_response_additional_headers,
|
||||
|
|
@ -85,7 +87,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
|
|||
batch_response_view,
|
||||
canonical_provider_batch_id,
|
||||
)
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
|
||||
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
|
||||
from litellm.proxy.spend_tracking.input_tokens import (
|
||||
inline_pdf_file_count,
|
||||
models_counted_for,
|
||||
|
|
@ -745,6 +747,7 @@ class RequestRateLimiterStash:
|
|||
batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
|
||||
reservation_released: bool = False
|
||||
tpm_limited_tags: frozenset[str] = field(default_factory=frozenset)
|
||||
socket_agent_windows: tuple[RateLimitDescriptor, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -864,6 +867,62 @@ def _call_id_from_callback_kwargs(kwargs: object) -> str | None:
|
|||
return call_id if isinstance(call_id, str) else None
|
||||
|
||||
|
||||
def _socket_connection_call_id(kwargs: Mapping[str, object]) -> object:
|
||||
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params"))
|
||||
litellm_metadata: Final = None if litellm_params is None else as_str_mapping(litellm_params.get("litellm_metadata"))
|
||||
return None if litellm_metadata is None else litellm_metadata.get(RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY)
|
||||
|
||||
|
||||
def _is_socket_turn(kwargs: Mapping[str, object]) -> bool:
|
||||
if kwargs.get("call_type") == CallTypes.aresponses_websocket.value:
|
||||
return True
|
||||
connection_stash: Final = get_request_stash()
|
||||
connection_call_id: Final = _socket_connection_call_id(kwargs)
|
||||
return (
|
||||
connection_stash is not None
|
||||
and connection_call_id is not None
|
||||
and connection_call_id == connection_stash.owner_litellm_call_id
|
||||
)
|
||||
|
||||
|
||||
_AGENT_DESCRIPTOR_KEYS: Final = frozenset({"agent", "agent_session"})
|
||||
|
||||
_SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS: Final = frozenset(
|
||||
{PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, PTU_TEAM_DESCRIPTOR_KEY}
|
||||
)
|
||||
|
||||
|
||||
def _socket_turn_tpm_windows(descriptors: Sequence[RateLimitDescriptor]) -> tuple[RateLimitDescriptor, ...]:
|
||||
return tuple(
|
||||
RateLimitDescriptor(
|
||||
key=descriptor["key"],
|
||||
value=descriptor["value"],
|
||||
rate_limit=RateLimitDescriptorRateLimitObject(
|
||||
tokens_per_unit=tokens_per_unit, window_size=(descriptor.get("rate_limit") or {}).get("window_size")
|
||||
),
|
||||
)
|
||||
for descriptor in descriptors
|
||||
if descriptor["key"] not in _SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS
|
||||
and (tokens_per_unit := (descriptor.get("rate_limit") or {}).get("tokens_per_unit")) is not None
|
||||
)
|
||||
|
||||
|
||||
def _unmodeled_response_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
|
||||
combined_usage: Final = kwargs.get("combined_usage_object")
|
||||
if isinstance(combined_usage, Usage):
|
||||
return combined_usage
|
||||
return socket_session_usage(kwargs, response_obj)
|
||||
|
||||
|
||||
def _tpm_limited_tags(call_stash: RequestRateLimiterStash | None, kwargs: Mapping[str, object]) -> frozenset[str]:
|
||||
if call_stash is not None:
|
||||
return call_stash.tpm_limited_tags
|
||||
connection_stash: Final = get_request_stash()
|
||||
if connection_stash is None or not _is_socket_turn(kwargs):
|
||||
return frozenset()
|
||||
return connection_stash.tpm_limited_tags
|
||||
|
||||
|
||||
def _parse_output_cap_value(raw_value: object) -> int | None:
|
||||
if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float, str)):
|
||||
return None
|
||||
|
|
@ -2967,6 +3026,40 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if response["overall_code"] == "OVER_LIMIT":
|
||||
self._handle_rate_limit_error(response, descriptors, requested_model)
|
||||
|
||||
@with_service_target("rate_limits")
|
||||
async def enforce_token_limits_for_frame(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
|
||||
) -> None:
|
||||
connection_stash: Final = get_request_stash()
|
||||
tags: Final = sorted(connection_stash.tpm_limited_tags) if connection_stash is not None else []
|
||||
agent_windows: Final = connection_stash.socket_agent_windows if connection_stash is not None else ()
|
||||
hidden_stash: Final = _request_stash.set(None)
|
||||
try:
|
||||
descriptors: Final = await self._build_request_rate_limit_descriptors(
|
||||
user_api_key_dict,
|
||||
MappingProxyType({"model": requested_model, "metadata": {"tags": tags}}),
|
||||
CallTypes.aresponses_websocket.value,
|
||||
)
|
||||
finally:
|
||||
_request_stash.reset(hidden_stash)
|
||||
built_scopes: Final = frozenset((d["key"], d["value"]) for d in descriptors)
|
||||
tpm_windows: Final = _socket_turn_tpm_windows(
|
||||
(*descriptors, *(d for d in agent_windows if (d["key"], d["value"]) not in built_scopes))
|
||||
)
|
||||
try:
|
||||
response: Final = await self.should_rate_limit(
|
||||
tpm_windows,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
read_only=True,
|
||||
skip_parallel_check=True,
|
||||
)
|
||||
except RateLimitUnverifiableError as unverifiable:
|
||||
raise ServiceUnavailableError(
|
||||
message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model or ""
|
||||
) from unverifiable
|
||||
if response["overall_code"] == "OVER_LIMIT":
|
||||
self._handle_rate_limit_error(response, tpm_windows, requested_model)
|
||||
|
||||
def _rate_limited_model(self, requested_model: str | None) -> RateLimitedModel | None:
|
||||
if not requested_model:
|
||||
return None
|
||||
|
|
@ -3777,7 +3870,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
def _handle_rate_limit_error(
|
||||
self,
|
||||
response: RateLimitResponse,
|
||||
descriptors: list[RateLimitDescriptor],
|
||||
descriptors: Sequence[RateLimitDescriptor],
|
||||
requested_model: str | None = None,
|
||||
) -> None:
|
||||
"""Handle rate limit exceeded by raising :class:`ProxyRateLimitError` (a 429)."""
|
||||
|
|
@ -4329,6 +4422,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
for d in descriptors
|
||||
if d["key"] == "tag" and d["rate_limit"] is not None and d["rate_limit"].get("tokens_per_unit") is not None
|
||||
)
|
||||
stash.socket_agent_windows = tuple(d for d in descriptors if d["key"] in _AGENT_DESCRIPTOR_KEYS)
|
||||
|
||||
# Only check rate limits if we have descriptors with actual limits
|
||||
if descriptors:
|
||||
|
|
@ -5286,9 +5380,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
# Get total tokens from response. Responses LiteLLM does not model
|
||||
# (e.g. pass-through, whose usage is reported by the upstream rather
|
||||
# than parsed out of the body) carry their usage in
|
||||
# ``combined_usage_object`` instead, and would otherwise never charge
|
||||
# the TPM window.
|
||||
_combined_usage: Final = kwargs.get("combined_usage_object")
|
||||
# ``combined_usage_object`` instead (a Responses WebSocket turn in its
|
||||
# raw events) and would otherwise never charge the TPM window.
|
||||
_usage: Final[Usage | dict[str, object] | None] = (
|
||||
getattr(response_obj, "usage", None)
|
||||
if isinstance(
|
||||
|
|
@ -5302,9 +5395,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
else get_usage_object(response_obj)
|
||||
if isinstance(response_obj, ExtractionResponse)
|
||||
else _combined_usage
|
||||
if isinstance(_combined_usage, Usage)
|
||||
else None
|
||||
else _unmodeled_response_usage(kwargs, response_obj)
|
||||
)
|
||||
total_tokens = self._get_total_tokens_from_usage(usage=_usage, rate_limit_type=rate_limit_type)
|
||||
if total_tokens == 0:
|
||||
|
|
@ -5337,7 +5428,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
targets: Final = self._collect_tpm_scope_targets(
|
||||
standard_logging_metadata=standard_logging_metadata,
|
||||
kwargs=kwargs,
|
||||
tpm_limited_tags=stash.tpm_limited_tags if stash is not None else frozenset(),
|
||||
tpm_limited_tags=_tpm_limited_tags(stash, kwargs),
|
||||
model_group=reconcile_model.group if reconcile_model is not None else None,
|
||||
)
|
||||
targets.extend(
|
||||
|
|
@ -5365,16 +5456,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
reserved_tokens=reserved_tokens,
|
||||
)
|
||||
)
|
||||
ptu_skips_socket_usage: Final = kwargs.get("call_type") == CallTypes.aresponses_websocket.value
|
||||
pipeline_operations.extend(
|
||||
self._build_team_ptu_tpm_ops(
|
||||
standard_logging_metadata, # pyright: ignore[reportUnknownArgumentType] # untyped logging metadata
|
||||
usage_source=response_obj if _usage is None else _usage,
|
||||
usage_source=response_obj if _usage is None or ptu_skips_socket_usage else _usage,
|
||||
reconcile_model=reconcile_model,
|
||||
reserved_scopes=reserved_scopes,
|
||||
reserved_ceiling=stash.ptu_ceiling if stash is not None else None,
|
||||
reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0,
|
||||
reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(),
|
||||
total_tokens=total_tokens,
|
||||
total_tokens=0 if ptu_skips_socket_usage else total_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -5469,27 +5561,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
else None,
|
||||
)
|
||||
|
||||
pipeline_operations: Final = self._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
rate_limit_type=rate_limit_type,
|
||||
await self._apply_usage_operations(
|
||||
self._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
rate_limit_type=rate_limit_type,
|
||||
),
|
||||
litellm_parent_otel_span,
|
||||
)
|
||||
unguarded_operations: Final = [
|
||||
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
|
||||
]
|
||||
window_guarded_operations: Final = tuple(
|
||||
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
|
||||
)
|
||||
if unguarded_operations:
|
||||
await self.async_increment_tokens_with_ttl_preservation(
|
||||
pipeline_operations=unguarded_operations,
|
||||
parent_otel_span=litellm_parent_otel_span,
|
||||
)
|
||||
if window_guarded_operations:
|
||||
await self.async_increment_reservation_aware_tokens(
|
||||
pipeline_operations=window_guarded_operations,
|
||||
parent_otel_span=litellm_parent_otel_span,
|
||||
)
|
||||
io_token_operations: Final = self._build_io_token_reservation_ops(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
|
|
@ -5509,6 +5588,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
|
||||
|
||||
async def _apply_usage_operations(
|
||||
self, pipeline_operations: Sequence[RedisPipelineIncrementOperation], parent_otel_span: Span | None
|
||||
) -> None:
|
||||
unguarded_operations: Final = [
|
||||
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
|
||||
]
|
||||
window_guarded_operations: Final = tuple(
|
||||
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
|
||||
)
|
||||
if unguarded_operations:
|
||||
await self.async_increment_tokens_with_ttl_preservation(
|
||||
pipeline_operations=unguarded_operations,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
if window_guarded_operations:
|
||||
await self.async_increment_reservation_aware_tokens(
|
||||
pipeline_operations=window_guarded_operations,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
|
||||
@with_service_target("rate_limits")
|
||||
async def async_logging_hook(
|
||||
self,
|
||||
|
|
@ -5622,11 +5721,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
|
||||
try:
|
||||
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(kwargs)
|
||||
failed_call: Final = _callback_kwargs_mapping(kwargs)
|
||||
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(failed_call)
|
||||
|
||||
pipeline_operations: Final[list[RedisPipelineIncrementOperation]] = []
|
||||
|
||||
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
|
||||
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(failed_call))
|
||||
await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span, in_logging_callback=True)
|
||||
|
||||
# Skip the reservation refund if async_post_call_failure_hook
|
||||
|
|
@ -5638,7 +5738,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if stash is None or stash.reservation_released
|
||||
else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
|
||||
)
|
||||
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(kwargs)
|
||||
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call)
|
||||
|
||||
if stash is not None and reserved_tokens > 0:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -5722,6 +5822,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
)
|
||||
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
|
||||
stash.reservation_released = True
|
||||
if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call):
|
||||
await self._apply_usage_operations(
|
||||
self._build_success_event_pipeline_operations(
|
||||
kwargs=dict(failed_call), response_obj=None, rate_limit_type=self.get_rate_limit_type()
|
||||
),
|
||||
litellm_parent_otel_span,
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in rate limit failure event: %s", e)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,11 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
get_metadata_variable_name_from_kwargs,
|
||||
get_parent_otel_span_from_kwargs,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
UsageCaptureObservation,
|
||||
UsageObservation,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
|
||||
from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS
|
||||
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
|
||||
|
|
@ -63,6 +67,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 #
|
|||
sanitize_error_information_for_spend_logs,
|
||||
should_store_prompts_and_responses_in_spend_logs,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs
|
||||
from litellm.proxy.utils import (
|
||||
REJECTED_BEFORE_REQUEST_SETUP,
|
||||
REJECTED_BEFORE_REQUEST_SETUP_KEY,
|
||||
|
|
@ -344,11 +349,27 @@ def _attempt_spend_row_kwargs(
|
|||
"litellm_params": litellm_params,
|
||||
"combined_usage_object": attempt.usage,
|
||||
"response_cost": attempt.response_cost,
|
||||
"_accounting_usage_observed": UsageObservation.ESTIMATED,
|
||||
"_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
|
||||
attempt.usage, UsageObservation.ESTIMATED, attempt.response_cost
|
||||
),
|
||||
"_accounting_prior_captures": (),
|
||||
}
|
||||
row.pop("billed_failed_attempts", None)
|
||||
row.pop("_accounting_capture", None)
|
||||
call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id")
|
||||
if isinstance(call_id, str):
|
||||
row["litellm_call_id"] = f"{call_id}{suffix}"
|
||||
row["parent_request_id"] = call_id
|
||||
metadata["parent_request_id"] = call_id
|
||||
captured_attempts: Final = request_data.get("_accounting_billed_attempts")
|
||||
if isinstance(captured_attempts, tuple):
|
||||
for captured in TypeAdapter(tuple[object, ...]).validate_python(captured_attempts):
|
||||
if (
|
||||
isinstance(captured, CapturedUsage)
|
||||
and captured.payload["litellm_call_id"] == row["litellm_call_id"]
|
||||
):
|
||||
row["_accounting_capture"] = captured
|
||||
standard_logging_object: Final = request_data.get("standard_logging_object")
|
||||
if isinstance(standard_logging_object, dict):
|
||||
payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict
|
||||
|
|
@ -385,7 +406,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
if self.spend_event_producer is None or not is_offloadable_success(response_obj):
|
||||
await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time)
|
||||
await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time)
|
||||
return
|
||||
event: Final = build_spend_event(
|
||||
kwargs,
|
||||
|
|
@ -396,7 +417,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
)
|
||||
if isinstance(event, SpendEventBuildError):
|
||||
verbose_proxy_logger.warning("collector: tracking cost in-process, event not buildable: %s", event.reason)
|
||||
await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time)
|
||||
await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time)
|
||||
return
|
||||
await self.spend_event_producer.publish(event)
|
||||
|
||||
|
|
@ -434,6 +455,9 @@ class _ProxyDBLogger(CustomLogger):
|
|||
"Failed to invalidate budget reservation counters after failure release failed"
|
||||
)
|
||||
|
||||
capture_metadata: Final = await self._try_capture_failure_usage(
|
||||
request_data, original_exception, user_api_key_dict
|
||||
)
|
||||
if completed_cost and tracking_disabled:
|
||||
return
|
||||
|
||||
|
|
@ -477,8 +501,12 @@ class _ProxyDBLogger(CustomLogger):
|
|||
)
|
||||
_metadata["error_information"] = _error_information
|
||||
|
||||
_metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
metadata=_metadata, original_exception=original_exception
|
||||
_metadata = ( # rebind-ok: enriched identity is shared with the earlier immutable capture
|
||||
{**_metadata, **capture_metadata}
|
||||
if capture_metadata is not None
|
||||
else await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled(
|
||||
metadata=_metadata, original_exception=original_exception
|
||||
)
|
||||
)
|
||||
|
||||
request_fields: Final = _str_keyed_fields(request_data)
|
||||
|
|
@ -595,6 +623,106 @@ class _ProxyDBLogger(CustomLogger):
|
|||
),
|
||||
)
|
||||
|
||||
async def _failure_capture_metadata(
|
||||
self, request_data: Mapping[str, object], exception: Exception, auth: UserAPIKeyAuth
|
||||
) -> Mapping[str, object] | None:
|
||||
from litellm.proxy.proxy_server import usage_capture
|
||||
|
||||
if usage_capture.meter is None:
|
||||
return None
|
||||
captured_identity: Final = (
|
||||
_batch_accounting_for(request_data.get("litellm_call_id")) is not None
|
||||
and request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is not REJECTED_BEFORE_REQUEST_SETUP
|
||||
)
|
||||
current_identity: Final = {
|
||||
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
|
||||
"user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
|
||||
}
|
||||
metadata: Final = (
|
||||
{
|
||||
**dict.fromkeys(current_identity),
|
||||
**{
|
||||
key: value
|
||||
for key, value in _REQUEST_DATA.validate_python(
|
||||
get_litellm_metadata_from_kwargs(dict(request_data))
|
||||
).items()
|
||||
if key in current_identity or key == "tags"
|
||||
},
|
||||
}
|
||||
if captured_identity
|
||||
else current_identity
|
||||
)
|
||||
return await self._enrich_failure_metadata_unless_db_stalled(
|
||||
metadata=metadata, original_exception=exception, resolve_missing_key_identity=not captured_identity
|
||||
)
|
||||
|
||||
async def _try_capture_failure_usage(
|
||||
self,
|
||||
request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer
|
||||
original_exception: Exception,
|
||||
auth: UserAPIKeyAuth,
|
||||
) -> Mapping[str, object] | None:
|
||||
try:
|
||||
metadata: Final = await self._failure_capture_metadata(request_data, original_exception, auth)
|
||||
self._capture_failure_usage(request_data, original_exception, auth, metadata)
|
||||
except Exception: # noqa: BLE001 # capture failure must not suppress legacy billing or reservation release
|
||||
verbose_proxy_logger.exception("Usage capture failed on the request failure path")
|
||||
return None
|
||||
return metadata
|
||||
|
||||
def _capture_failure_usage(
|
||||
self,
|
||||
request_data: dict[str, object], # mutable-ok: receipt handoff shared with the legacy writer
|
||||
original_exception: Exception,
|
||||
auth: UserAPIKeyAuth,
|
||||
metadata: Mapping[str, object] | None = None,
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.db.exception_handler import exception_chain
|
||||
from litellm.proxy.proxy_server import usage_capture
|
||||
from litellm.proxy.spend_tracking.usage_capture import UsageAcknowledgementError
|
||||
|
||||
if any(isinstance(error, UsageAcknowledgementError) for error in exception_chain(original_exception)):
|
||||
return
|
||||
usage_meter: Final = usage_capture.meter
|
||||
|
||||
if usage_meter is None:
|
||||
return
|
||||
logging_obj: Final = request_data.get("litellm_logging_obj")
|
||||
details: Final[Mapping[str, object]] = (
|
||||
_str_keyed_fields(logging_obj.model_call_details) if isinstance(logging_obj, Logging) else {}
|
||||
)
|
||||
params: Final = _str_keyed_fields(request_data.get("litellm_params") or details.get("litellm_params"))
|
||||
failure_data: Final = {
|
||||
**details,
|
||||
**request_data,
|
||||
"litellm_params": {
|
||||
**params,
|
||||
"metadata": {
|
||||
**_str_keyed_fields(params.get("metadata")),
|
||||
**(
|
||||
metadata
|
||||
if metadata is not None
|
||||
else {
|
||||
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
|
||||
"user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
|
||||
}
|
||||
),
|
||||
"status": "failure",
|
||||
},
|
||||
},
|
||||
}
|
||||
start: Final = logging_obj.start_time if isinstance(logging_obj, Logging) else request_data.get("start_time")
|
||||
if not isinstance(start, datetime):
|
||||
return
|
||||
end: Final = datetime.now(timezone.utc)
|
||||
captured: Final = tuple(
|
||||
capture_callback(usage_meter, row, original_exception, start, end)
|
||||
for _, row in _failure_spend_rows(failure_data, 0.0, None, self._deployment_lookup)
|
||||
)
|
||||
request_data["_accounting_capture"] = captured[0] # rebind-ok: share one receipt with the legacy writer
|
||||
request_data["_accounting_billed_attempts"] = captured[1:] # rebind-ok: share attempt receipts
|
||||
|
||||
async def _write_failure_spend_rows(
|
||||
self,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
@ -780,6 +908,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
resolve_missing_key_identity=str(kwargs.get("call_type")) not in _CAPTURED_IDENTITY_CALL_TYPES,
|
||||
)
|
||||
_write_spend_metadata_to_kwargs(kwargs=kwargs, metadata=metadata)
|
||||
kwargs.update(await captured_callback_kwargs(kwargs, completion_response, start_time, end_time))
|
||||
budget_reservation: Final = _get_budget_reservation_from_metadata(metadata=metadata)
|
||||
if accounting is not None and (budget_reservation is not None or litellm.apply_search_free_tier):
|
||||
accounting.write_started = True
|
||||
|
|
@ -967,11 +1096,13 @@ class _ProxyDBLogger(CustomLogger):
|
|||
|
||||
@staticmethod
|
||||
async def _enrich_failure_metadata_unless_db_stalled(
|
||||
metadata: dict[str, object], original_exception: Exception
|
||||
metadata: dict[str, object], original_exception: Exception, resolve_missing_key_identity: bool = True
|
||||
) -> dict[str, object]:
|
||||
if isinstance(original_exception, DBLookupDeadlineExceeded):
|
||||
return metadata
|
||||
return await ProxyDBLogger._enrich_failure_metadata_with_key_info(metadata=metadata)
|
||||
return await ProxyDBLogger._enrich_failure_metadata_with_key_info(
|
||||
metadata=metadata, resolve_missing_key_identity=resolve_missing_key_identity
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _enrich_failure_metadata_with_key_info(metadata: dict, resolve_missing_key_identity: bool = True) -> dict:
|
||||
|
|
|
|||
|
|
@ -2,14 +2,32 @@
|
|||
Shared utility functions for rate limiter hooks.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK
|
||||
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
|
||||
from litellm.litellm_core_utils.core_helpers import iter_items
|
||||
from litellm.types.router import ModelGroupInfo
|
||||
from litellm.types.utils import PriorityReservationDict
|
||||
from litellm.types.utils import CallTypes, PriorityReservationDict, Usage
|
||||
|
||||
_SOCKET_SESSION_EVENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def socket_session_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
|
||||
if not isinstance(response_obj, list):
|
||||
return None
|
||||
match kwargs.get("call_type"):
|
||||
case CallTypes.aresponses_websocket.value:
|
||||
return ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
|
||||
_SOCKET_SESSION_EVENTS.validate_python(response_obj)
|
||||
)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def resolve_llm_provider_for_rate_limit(
|
||||
|
|
|
|||
|
|
@ -64,11 +64,16 @@ from litellm.llms.azure.passthrough.transformation import (
|
|||
from litellm.llms.bedrock.passthrough.transformation import is_bedrock_streaming_endpoint
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.llms.deepgram.common_utils import (
|
||||
DeepgramSpeakMeter,
|
||||
deepgram_listen_callback_params,
|
||||
deepgram_listen_is_priced,
|
||||
deepgram_listen_registry_key,
|
||||
deepgram_listen_requested_model,
|
||||
deepgram_listen_websocket_target,
|
||||
deepgram_speak_character_rate,
|
||||
deepgram_speak_registry_key,
|
||||
deepgram_speak_requested_model,
|
||||
deepgram_speak_websocket_target,
|
||||
)
|
||||
from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base
|
||||
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
|
||||
|
|
@ -129,6 +134,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
|||
YieldsToConfiguredPassThroughRoute,
|
||||
create_pass_through_route,
|
||||
create_websocket_passthrough_route,
|
||||
truncated_close_reason,
|
||||
websocket_passthrough_request,
|
||||
)
|
||||
from litellm.proxy.utils import ProxyLogging as ProxyLoggingType
|
||||
|
|
@ -145,6 +151,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
|
||||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
WebSocketSessionMeter,
|
||||
)
|
||||
from litellm.types.passthrough_endpoints.tinyfish import (
|
||||
TINYFISH_AUTHENTICATED_RUN_FIELDS,
|
||||
|
|
@ -3460,6 +3467,7 @@ class _WebsocketRelay(Protocol):
|
|||
forward_headers: bool,
|
||||
endpoint: str,
|
||||
accept_websocket: bool,
|
||||
meter: WebSocketSessionMeter | None = None,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
|
|
@ -3621,6 +3629,54 @@ async def deepgram_listen_websocket_route(
|
|||
)
|
||||
|
||||
|
||||
async def deepgram_speak_user_api_key_auth(websocket: WebSocket) -> UserAPIKeyAuth:
|
||||
return await user_api_key_auth_websocket_for_model(
|
||||
websocket, model=deepgram_speak_requested_model(websocket.url.query)
|
||||
)
|
||||
|
||||
|
||||
@router.websocket("/deepgram/v1/speak")
|
||||
@router.websocket("/deepgram/speak")
|
||||
async def deepgram_speak_websocket_route(
|
||||
websocket: WebSocket,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(deepgram_speak_user_api_key_auth)],
|
||||
relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)],
|
||||
) -> None:
|
||||
deepgram_api_key: Final = passthrough_endpoint_router.get_credentials(
|
||||
custom_llm_provider=litellm.LlmProviders.DEEPGRAM.value,
|
||||
region_name=None,
|
||||
)
|
||||
if deepgram_api_key is None:
|
||||
await websocket.close(code=1011, reason=_DEEPGRAM_WS_MISSING_KEY_REASON)
|
||||
return
|
||||
|
||||
await websocket.accept(subprotocol=_negotiated_websocket_subprotocol(websocket))
|
||||
target: Final = deepgram_speak_websocket_target(
|
||||
api_base=get_secret_str("DEEPGRAM_API_BASE"),
|
||||
query_string=websocket.url.query,
|
||||
)
|
||||
character_rate: Final = deepgram_speak_character_rate(target)
|
||||
if character_rate is None:
|
||||
await websocket.close(
|
||||
code=1008,
|
||||
reason=truncated_close_reason(
|
||||
_DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_speak_registry_key(target))
|
||||
),
|
||||
)
|
||||
return
|
||||
|
||||
await relay(
|
||||
websocket=websocket,
|
||||
target=target,
|
||||
custom_headers={"Authorization": f"Token {deepgram_api_key}"},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
forward_headers=False,
|
||||
endpoint=websocket.url.path,
|
||||
accept_websocket=False,
|
||||
meter=DeepgramSpeakMeter(target, character_rate),
|
||||
)
|
||||
|
||||
|
||||
_OPENAI_BETA_HEADER: Final = "openai-beta"
|
||||
_CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"})
|
||||
|
||||
|
|
@ -4474,10 +4530,16 @@ async def watsonx_proxy_route(
|
|||
if provider_config is None:
|
||||
raise HTTPException(status_code=404, detail="Watsonx passthrough config not found")
|
||||
|
||||
watsonx_credentials: Final = passthrough_endpoint_router.get_proxy_wide_deployment_credentials(
|
||||
custom_llm_provider="watsonx"
|
||||
)
|
||||
api_key: Final = watsonx_credentials.api_key if watsonx_credentials is not None else None
|
||||
api_base: Final = watsonx_credentials.api_base if watsonx_credentials is not None else None
|
||||
|
||||
# Get complete URL with version parameter
|
||||
complete_url, _ = provider_config.get_complete_url(
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model="",
|
||||
endpoint=endpoint,
|
||||
request_query_params=None,
|
||||
|
|
@ -4491,8 +4553,8 @@ async def watsonx_proxy_route(
|
|||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
api_key=None,
|
||||
api_base=None,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
# Check for streaming
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough
|
|||
from litellm.proxy.pass_through_endpoints.success_handler import (
|
||||
PassThroughEndpointLogging,
|
||||
)
|
||||
from litellm.types.decisions import OpenAIDecisionResponse
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
||||
SETTLES_BACKGROUND_RESPONSE_KWARG,
|
||||
|
|
@ -277,6 +278,22 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
def is_openai_agents_route(url_route: str) -> bool:
|
||||
return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path)
|
||||
|
||||
@staticmethod
|
||||
def is_openai_decisions_route(url_route: str) -> bool:
|
||||
return is_openai_compatible_url(url_route) and urlparse(url_route).path.rstrip("/").endswith("/v1/decisions")
|
||||
|
||||
@staticmethod
|
||||
def handles_route(url_route: str) -> bool:
|
||||
return (
|
||||
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _unbilled_call_model(
|
||||
url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object]
|
||||
|
|
@ -479,8 +496,19 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
httpx_response.request.method, url_route, response_body
|
||||
)
|
||||
is_responses: Final = responses_billing != "unbilled"
|
||||
is_decisions: Final = (
|
||||
httpx_response.request.method == "POST"
|
||||
and OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
|
||||
)
|
||||
|
||||
if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses):
|
||||
if not (
|
||||
is_chat_completions
|
||||
or is_embeddings
|
||||
or is_image_generation
|
||||
or is_image_editing
|
||||
or is_responses
|
||||
or is_decisions
|
||||
):
|
||||
OpenAIPassthroughLoggingHandler._label_unbilled_call(
|
||||
logging_obj,
|
||||
url_route,
|
||||
|
|
@ -512,7 +540,13 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
try:
|
||||
response_cost = 0.0
|
||||
litellm_model_response: (
|
||||
ModelResponse | TextCompletionResponse | EmbeddingResponse | ImageResponse | ResponsesAPIResponse | None
|
||||
ModelResponse
|
||||
| TextCompletionResponse
|
||||
| EmbeddingResponse
|
||||
| ImageResponse
|
||||
| ResponsesAPIResponse
|
||||
| OpenAIDecisionResponse
|
||||
| None
|
||||
) = None
|
||||
handler_instance: Final = OpenAIPassthroughLoggingHandler()
|
||||
|
||||
|
|
@ -564,11 +598,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
response_body=response_body,
|
||||
request_body=request_body,
|
||||
)
|
||||
# Mark call type for downstream image-aware logic/metrics
|
||||
try:
|
||||
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
|
||||
except Exception:
|
||||
pass
|
||||
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
|
||||
# Create a simple response object for logging
|
||||
litellm_model_response = ImageResponse(
|
||||
data=response_body.get("data", []),
|
||||
|
|
@ -583,11 +613,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
response_body=response_body,
|
||||
request_body=request_body,
|
||||
)
|
||||
# Mark call type for downstream image-aware logic/metrics
|
||||
try:
|
||||
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
|
||||
except Exception:
|
||||
pass
|
||||
logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
|
||||
# Create a simple response object for logging
|
||||
litellm_model_response = ImageResponse(
|
||||
data=response_body.get("data", []),
|
||||
|
|
@ -609,6 +635,14 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
elif is_decisions:
|
||||
litellm_model_response = OpenAIDecisionResponse.model_validate(response_body) # rebind-ok: elif-chain
|
||||
response_cost = litellm.completion_cost( # rebind-ok: elif-chain
|
||||
completion_response=litellm_model_response,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
call_type="decisions",
|
||||
)
|
||||
|
||||
# Update kwargs with cost information
|
||||
kwargs["response_cost"] = response_cost
|
||||
|
|
@ -656,6 +690,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
if is_image_generation
|
||||
else "image_editing"
|
||||
if is_image_editing
|
||||
else "decisions"
|
||||
if is_decisions
|
||||
else "responses"
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
|
|||
|
|
@ -2584,7 +2584,7 @@ def _json_object_frame(frame: str | bytes) -> dict[str, object] | None:
|
|||
return decoded if isinstance(decoded, dict) else None
|
||||
|
||||
|
||||
def _truncated_close_reason(reason: str) -> str:
|
||||
def truncated_close_reason(reason: str) -> str:
|
||||
"""
|
||||
Fit a close reason inside the byte budget a WebSocket close frame allows, without splitting a character
|
||||
"""
|
||||
|
|
@ -2958,7 +2958,7 @@ async def websocket_passthrough_request(
|
|||
if upstream_close is not None and _client_socket_is_open(websocket):
|
||||
await websocket.close(
|
||||
code=upstream_close.code,
|
||||
reason=_truncated_close_reason(upstream_close.reason),
|
||||
reason=truncated_close_reason(upstream_close.reason),
|
||||
)
|
||||
|
||||
end_time: Final = datetime.now()
|
||||
|
|
|
|||
|
|
@ -132,6 +132,19 @@ class PassthroughEndpointRouter:
|
|||
api_base=deployment_credentials.api_base if deployment_credentials is not None else None,
|
||||
)
|
||||
|
||||
def get_proxy_wide_deployment_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None:
|
||||
deployment_credentials: Final = self._get_deployment_credentials(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
region_name=None,
|
||||
include_team_deployments=False,
|
||||
)
|
||||
if deployment_credentials is None:
|
||||
return None
|
||||
api_key: Final = _resolve_oidc_reference(deployment_credentials.api_key)
|
||||
if not api_key:
|
||||
return None
|
||||
return _PassThroughCredentials(api_key=api_key, api_base=deployment_credentials.api_base)
|
||||
|
||||
def has_credentials(self, custom_llm_provider: str) -> bool:
|
||||
if (
|
||||
self._get_deployment_credentials(
|
||||
|
|
|
|||
|
|
@ -666,28 +666,11 @@ class PassThroughEndpointLogging:
|
|||
return False
|
||||
|
||||
def _is_supported_openai_endpoint(self, url_route: str) -> bool:
|
||||
"""Check if the OpenAI endpoint is supported by the passthrough logging handler.
|
||||
|
||||
The Responses API route is included because
|
||||
`openai_passthrough_handler` has a dedicated `elif is_responses:`
|
||||
branch that knows how to extract usage + cost from the
|
||||
Responses-API on-the-wire shape. Without including it here, the
|
||||
outer dispatch filters Responses calls out before reaching the
|
||||
handler — the inner branch is then unreachable and Responses
|
||||
calls land in `LiteLLM_SpendLogs` with zero tokens / zero spend.
|
||||
"""
|
||||
from .llm_provider_handlers.openai_passthrough_logging_handler import (
|
||||
OpenAIPassthroughLoggingHandler,
|
||||
)
|
||||
|
||||
return (
|
||||
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
|
||||
or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
|
||||
)
|
||||
return OpenAIPassthroughLoggingHandler.handles_route(url_route)
|
||||
|
||||
def _set_cost_per_request(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -473,7 +473,6 @@ from litellm.proxy.common_request_processing import ( # noqa: F401, RUF100 # l
|
|||
log_llm_api_exception,
|
||||
open_sse_before_first_byte,
|
||||
proxy_exception_from_route_error,
|
||||
request_litellm_call_id,
|
||||
resolve_litellm_call_id,
|
||||
should_return_raw_model_name,
|
||||
sse_error_payload,
|
||||
|
|
@ -840,11 +839,15 @@ from litellm.proxy.spend_tracking.spend_event_producer import (
|
|||
SpendEventProducer,
|
||||
build_spend_event_producer,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime
|
||||
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
|
||||
from litellm.proxy.telemetry.endpoints import router as telemetry_router
|
||||
from litellm.proxy.telemetry.middleware import TelemetryMiddleware
|
||||
from litellm.proxy.telemetry.runtime import TelemetryRuntime
|
||||
from litellm.proxy.telemetry.settings import describe_errors as describe_telemetry_errors
|
||||
from litellm.proxy.telemetry.settings import load_settings as load_telemetry_settings
|
||||
from litellm.router import router_arguments
|
||||
from litellm.router_config import build_router_config
|
||||
|
||||
try:
|
||||
from litellm.proxy.enterprise_billing.billing_metrics import (
|
||||
|
|
@ -1329,6 +1332,7 @@ async def _flush_spend_around_logging_drain_on_shutdown(
|
|||
finally:
|
||||
await worker.stop(discard_pending=True, timeout=GracefulShutdownManager.remaining(2.0, clock=clock))
|
||||
await drain_tasks(GracefulShutdownManager.remaining(5.0, clock=clock))
|
||||
await usage_capture.stop(GracefulShutdownManager.remaining(clock=clock))
|
||||
await flush_spend()
|
||||
|
||||
|
||||
|
|
@ -3202,9 +3206,39 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False):
|
|||
|
||||
spend_event_producer: SpendEventProducer | None = None
|
||||
|
||||
usage_capture: Final = UsageCaptureRuntime()
|
||||
|
||||
|
||||
def _configure_usage_capture() -> None:
|
||||
settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings)
|
||||
capture_settings: Final = settings.get("accounting_capture")
|
||||
if settings.get("accounting_sync_ack") and capture_settings is None:
|
||||
raise ValueError("accounting_sync_ack requires accounting_capture")
|
||||
if capture_settings is None:
|
||||
if usage_capture.meter is not None:
|
||||
raise ValueError("Disabling accounting_capture requires a worker restart")
|
||||
return
|
||||
if (
|
||||
prisma_client is None
|
||||
or CollectorSettings().enabled
|
||||
or os.getenv("SPEND_LOGS_URL")
|
||||
or litellm.apply_search_free_tier
|
||||
):
|
||||
raise ValueError(
|
||||
"Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier"
|
||||
)
|
||||
from litellm.proxy.db.accounting_ledger import AccountingLedger
|
||||
|
||||
ledger: Final = AccountingLedger.for_client(prisma_client)
|
||||
writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch)
|
||||
usage_capture.start(
|
||||
writer.write, UsageCaptureSettings.model_validate(capture_settings), load=ledger.load_usage_batch
|
||||
)
|
||||
|
||||
|
||||
def cost_tracking():
|
||||
global prisma_client, spend_event_producer
|
||||
_configure_usage_capture()
|
||||
if prisma_client is not None:
|
||||
from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
|
||||
|
||||
|
|
@ -7375,20 +7409,24 @@ class ProxyConfig:
|
|||
verbose_proxy_logger.warning(
|
||||
"Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k
|
||||
)
|
||||
router = litellm.Router(
|
||||
**router_params,
|
||||
assistants_config=assistants_config,
|
||||
search_tools=search_tools,
|
||||
router_general_settings=RouterGeneralSettings(
|
||||
async_only_mode=True # only init async clients
|
||||
),
|
||||
ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid
|
||||
fallback_access_check=router_fallback_access_check,
|
||||
fallback_budget_check=RouterFallbackBudgetCheck(
|
||||
is_enforced=router_fallback_budget_check.is_enforced,
|
||||
team_member_model_budget_limiter=model_max_budget_limiter,
|
||||
),
|
||||
auto_router_capability_limit=_license_check.auto_router_capability_limit,
|
||||
router = litellm.Router.from_config(
|
||||
build_router_config(
|
||||
router_arguments(
|
||||
**router_params,
|
||||
assistants_config=assistants_config,
|
||||
search_tools=search_tools,
|
||||
router_general_settings=RouterGeneralSettings(
|
||||
async_only_mode=True # only init async clients
|
||||
),
|
||||
ignore_invalid_deployments=True, # don't raise an error if a deployment is invalid
|
||||
fallback_access_check=router_fallback_access_check,
|
||||
fallback_budget_check=RouterFallbackBudgetCheck(
|
||||
is_enforced=router_fallback_budget_check.is_enforced,
|
||||
team_member_model_budget_limiter=model_max_budget_limiter,
|
||||
),
|
||||
auto_router_capability_limit=_license_check.auto_router_capability_limit,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if redis_usage_cache is not None and router.health().store.redis_cache is None:
|
||||
|
|
@ -7878,19 +7916,23 @@ class ProxyConfig:
|
|||
# Router can function with model_list=[] if search_tools are configured
|
||||
if len(_model_list) > 0 or search_tools:
|
||||
verbose_proxy_logger.debug("_model_list: %s", _model_list)
|
||||
llm_router = litellm.Router(
|
||||
model_list=_model_list,
|
||||
router_general_settings=RouterGeneralSettings(
|
||||
async_only_mode=True # only init async clients
|
||||
),
|
||||
search_tools=search_tools,
|
||||
ignore_invalid_deployments=True,
|
||||
fallback_access_check=router_fallback_access_check,
|
||||
fallback_budget_check=RouterFallbackBudgetCheck(
|
||||
is_enforced=router_fallback_budget_check.is_enforced,
|
||||
team_member_model_budget_limiter=model_max_budget_limiter,
|
||||
),
|
||||
auto_router_capability_limit=_license_check.auto_router_capability_limit,
|
||||
llm_router = litellm.Router.from_config(
|
||||
build_router_config(
|
||||
router_arguments(
|
||||
model_list=_model_list,
|
||||
router_general_settings=RouterGeneralSettings(
|
||||
async_only_mode=True # only init async clients
|
||||
),
|
||||
search_tools=search_tools,
|
||||
ignore_invalid_deployments=True,
|
||||
fallback_access_check=router_fallback_access_check,
|
||||
fallback_budget_check=RouterFallbackBudgetCheck(
|
||||
is_enforced=router_fallback_budget_check.is_enforced,
|
||||
team_member_model_budget_limiter=model_max_budget_limiter,
|
||||
),
|
||||
auto_router_capability_limit=_license_check.auto_router_capability_limit,
|
||||
)
|
||||
)
|
||||
)
|
||||
verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
|
||||
else:
|
||||
|
|
@ -13006,8 +13048,10 @@ async def completion(
|
|||
"""
|
||||
global user_temperature, user_request_timeout, user_max_tokens, user_api_base
|
||||
data = {}
|
||||
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
||||
try:
|
||||
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
base_llm_response_processor.data = data
|
||||
if user_api_key_dict is not None:
|
||||
if data.get("metadata") is None:
|
||||
data["metadata"] = {}
|
||||
|
|
@ -13021,7 +13065,6 @@ async def completion(
|
|||
data["metadata"]["user_api_key_org_alias"] = user_api_key_dict.organization_alias
|
||||
if hasattr(user_api_key_dict, "agent_id") and user_api_key_dict.agent_id is not None:
|
||||
data["metadata"]["agent_id"] = user_api_key_dict.agent_id
|
||||
base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
|
||||
return await base_llm_response_processor.base_process_llm_request(
|
||||
request=request,
|
||||
fastapi_response=fastapi_response,
|
||||
|
|
@ -13123,9 +13166,12 @@ async def completion(
|
|||
return _response
|
||||
except Exception as e:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=e,
|
||||
request_data=base_llm_response_processor.data,
|
||||
dispatched_call_type=base_llm_response_processor.route_type,
|
||||
)
|
||||
litellm_call_id: Final = request_litellm_call_id(data)
|
||||
litellm_call_id: Final = base_llm_response_processor.litellm_call_id
|
||||
log_llm_api_exception(e, litellm_call_id)
|
||||
if isinstance(e, ProxyException):
|
||||
raise with_litellm_call_id(e, litellm_call_id)
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError
|
||||
from litellm.exceptions import EndpointNotSupportedByModelError, RateLimitError
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException, guardrail_request_data_with_streaming
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
|
||||
|
|
@ -103,6 +103,7 @@ from litellm.responses.streaming_iterator import (
|
|||
ResponsesWsFrameGuard,
|
||||
ResponsesWsFrameRejection,
|
||||
spawn_logging_task,
|
||||
turn_refusal_frame,
|
||||
)
|
||||
from litellm.router import Router
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -2627,6 +2628,12 @@ async def _responses_websocket_session(
|
|||
route_type="_aresponses_websocket",
|
||||
skip_guardrails=True,
|
||||
)
|
||||
except RateLimitError as e:
|
||||
verbose_proxy_logger.info("Responses WebSocket refused at open by a rate limit: %s", e)
|
||||
with contextlib.suppress(WebSocketDisconnect, RuntimeError, OSError):
|
||||
await websocket.send_text(turn_refusal_frame(e))
|
||||
await websocket.close(code=1008, reason="Rate limit exceeded")
|
||||
return
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Responses WebSocket pre-call error")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.router import router_arguments
|
||||
from litellm.router_api import router_init_args
|
||||
from litellm.router_config import build_router_config
|
||||
|
||||
# Client-supplied params that make the router or the call path fabricate a
|
||||
# failure or a delay instead of calling the provider. The ``mock_testing_*``
|
||||
|
|
@ -98,7 +100,9 @@ async def _route_user_config_request(data: dict, route_type: str):
|
|||
filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
|
||||
|
||||
logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
|
||||
cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config))
|
||||
cached_router: Final = user_config_routers.get(
|
||||
filtered_config, lambda: litellm.Router.from_config(build_router_config(router_arguments(**filtered_config)))
|
||||
)
|
||||
cached_router.observe(logging_obj)
|
||||
return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj})
|
||||
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ async def _finish_finalization(
|
|||
if litellm.apply_search_free_tier
|
||||
else None
|
||||
)
|
||||
logging_obj.cost_allowance_allocation = allocation # rebind-ok: retain the allocation for durable capture
|
||||
logging_obj.cost_allowance_discount = ( # rebind-ok: retain allocation across snapshot retries
|
||||
allocation.discount if allocation is not None else 0.0
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import json
|
|||
import os
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime as dt
|
||||
from functools import reduce
|
||||
|
|
@ -55,6 +55,8 @@ from litellm.litellm_core_utils.get_llm_provider_logic import declared_authentic
|
|||
from litellm.litellm_core_utils.hidden_params import served_from_cache
|
||||
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
StandardLoggingPayloadSetup,
|
||||
UsageObservation,
|
||||
coerce_model_access_groups,
|
||||
is_valid_sha256_hash,
|
||||
request_model_access_groups_from_litellm_params,
|
||||
|
|
@ -599,6 +601,7 @@ def get_logging_payload(
|
|||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
llm_router: "Router | None" = None,
|
||||
capture_usage: Callable[[SpendLogsPayload, str | None], None] | None = None,
|
||||
) -> SpendLogsPayload:
|
||||
if kwargs is None:
|
||||
kwargs = {}
|
||||
|
|
@ -622,7 +625,9 @@ def get_logging_payload(
|
|||
routing_metadata,
|
||||
routing_litellm_params,
|
||||
)
|
||||
completion_start_time: Final = kwargs.get("completion_start_time", end_time)
|
||||
completion_start_time: Final = TypeAdapter[datetime | None](datetime | None).validate_python(
|
||||
kwargs.get("completion_start_time", end_time)
|
||||
)
|
||||
call_type: Final = kwargs.get("call_type")
|
||||
cache_hit: Final = kwargs.get("cache_hit", False)
|
||||
|
||||
|
|
@ -644,7 +649,7 @@ def get_logging_payload(
|
|||
_usage: Final = (
|
||||
chat_usage_of_batch.model_dump()
|
||||
if chat_usage_of_batch is not None
|
||||
else response_obj_dict.get("usage", None) or {}
|
||||
else StandardLoggingPayloadSetup.get_usage_as_dict(response_obj_dict)
|
||||
)
|
||||
if isinstance(_usage, litellm.Usage):
|
||||
usage = dict(_usage)
|
||||
|
|
@ -928,7 +933,9 @@ def get_logging_payload(
|
|||
cache_hit=str(cache_hit),
|
||||
startTime=_ensure_datetime_utc(start_time),
|
||||
endTime=_ensure_datetime_utc(end_time),
|
||||
completionStartTime=_ensure_datetime_utc(completion_start_time),
|
||||
completionStartTime=(
|
||||
_ensure_datetime_utc(completion_start_time) if completion_start_time is not None else None
|
||||
),
|
||||
model=model_name,
|
||||
user=metadata.get("user_api_key_user_id", "") or "",
|
||||
team_id=metadata.get("user_api_key_team_id", "") or "",
|
||||
|
|
@ -984,6 +991,14 @@ def get_logging_payload(
|
|||
payload.get("spend"),
|
||||
)
|
||||
|
||||
if capture_usage is not None:
|
||||
observed_usage: Final = (
|
||||
StandardLoggingPayloadSetup.get_usage_as_dict({"usage": usage})
|
||||
if usage and kwargs.get("_accounting_usage_observed") is not UsageObservation.UNKNOWN
|
||||
else None
|
||||
)
|
||||
capture_usage(payload, safe_dumps(observed_usage) if observed_usage is not None else None)
|
||||
|
||||
# Explicitly clear large intermediate objects to reduce memory pressure
|
||||
del response_obj_dict, usage, clean_metadata, additional_usage_values
|
||||
|
||||
|
|
|
|||
1084
litellm/proxy/spend_tracking/usage_capture.py
Normal file
1084
litellm/proxy/spend_tracking/usage_capture.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import Awaitable, Callable
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain, count, takewhile
|
||||
|
|
@ -114,6 +114,9 @@ class DurableUsageMeter:
|
|||
self._admission: Final = Lock()
|
||||
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
|
||||
self._wakeup: Final = asyncio.Event()
|
||||
self._capacity_changed: Final = asyncio.Event()
|
||||
self._reservations: Mapping[object, tuple[int, int]] = {}
|
||||
self._reserved_bytes = 0
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._closed = False
|
||||
self.pending_bytes = 0
|
||||
|
|
@ -143,6 +146,36 @@ class DurableUsageMeter:
|
|||
wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
|
||||
return wrapped
|
||||
|
||||
def reserve_producer(self, size: int, headroom: int) -> object:
|
||||
token: Final = object()
|
||||
self.resize_producer(token, size, headroom)
|
||||
return token
|
||||
|
||||
def resize_producer(self, token: object, size: int, headroom: int) -> None:
|
||||
with self._admission:
|
||||
previous: Final = self._reservations.get(token, (0, 0))[0]
|
||||
delta: Final = size - previous
|
||||
others: Final = (value[1] for key, value in self._reservations.items() if key is not token)
|
||||
required: Final = max((headroom, *others))
|
||||
if (
|
||||
self._closed
|
||||
or self.pending_bytes + delta > self.settings.max_pending_bytes
|
||||
or self._reserved_bytes + delta + required > self.settings.max_pending_bytes
|
||||
):
|
||||
raise UsageCaptureError("Batch capture producer byte capacity exhausted")
|
||||
self._reservations = {**self._reservations, token: (size, headroom)}
|
||||
self._reserved_bytes += delta
|
||||
self.pending_bytes += delta
|
||||
self._capacity_changed.set()
|
||||
|
||||
def release_producer(self, token: object) -> None:
|
||||
with self._admission:
|
||||
size, _ = self._reservations[token]
|
||||
self._reservations = {key: value for key, value in self._reservations.items() if key is not token}
|
||||
self._reserved_bytes -= size
|
||||
self.pending_bytes -= size
|
||||
self._capacity_changed.set()
|
||||
|
||||
def _ensure_worker(self) -> None:
|
||||
if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
|
||||
self._wakeup.set()
|
||||
|
|
@ -153,6 +186,34 @@ class DurableUsageMeter:
|
|||
envelope: Final = '{"event":' + event.model_dump_json() + "}"
|
||||
await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),)))
|
||||
|
||||
async def record_writes(self, writes: tuple[UsageWrite, ...]) -> None:
|
||||
size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
|
||||
capacity: Final = self.settings.max_pending_bytes - self._reserved_bytes
|
||||
if len(writes) > min(self.settings.batch_size, self.settings.max_pending_records) or (
|
||||
size > capacity and len(writes) > 1
|
||||
):
|
||||
middle: Final = len(writes) // 2
|
||||
await self.record_writes(writes[:middle])
|
||||
await self.record_writes(writes[middle:])
|
||||
return
|
||||
if size > capacity:
|
||||
raise UsageCaptureError("One usage observation exceeds capture capacity")
|
||||
while (receipt := self._submit_available(writes)) is None:
|
||||
self._wakeup.set()
|
||||
await self._capacity_changed.wait()
|
||||
self._wakeup.set()
|
||||
await asyncio.shield(receipt)
|
||||
|
||||
def _submit_available(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None] | None:
|
||||
self._capacity_changed.clear()
|
||||
try:
|
||||
receipt: Final = self.submit(writes)
|
||||
except UsageCaptureError:
|
||||
if self._closed or not self.pending_records or not writes:
|
||||
raise
|
||||
return None
|
||||
return receipt
|
||||
|
||||
async def flush(self) -> None:
|
||||
self._ensure_worker()
|
||||
self._wakeup.set()
|
||||
|
|
@ -240,6 +301,7 @@ class DurableUsageMeter:
|
|||
with self._admission:
|
||||
self.pending_bytes -= item.size
|
||||
self.pending_records -= len(item.writes)
|
||||
self._capacity_changed.set()
|
||||
if item.committed.done():
|
||||
continue
|
||||
if error is None:
|
||||
|
|
|
|||
|
|
@ -163,7 +163,12 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
from litellm.litellm_core_utils.deployment_pricing import pricing_models
|
||||
from litellm.litellm_core_utils.duration_parser import get_budget_window_start
|
||||
from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
BilledRoundsDispatch,
|
||||
Logging,
|
||||
UsageObservation,
|
||||
deployment_pricing_model_info,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
|
@ -1739,6 +1744,39 @@ def _failure_fields_to_lift(
|
|||
if _prompt_estimate is not None
|
||||
else ()
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation
|
||||
|
||||
_recorded_observation: Final[object] = _model_call_details.get("_accounting_usage_observed")
|
||||
_current_usage: Final = (
|
||||
_current_failure_usage_to_lift(
|
||||
_model_call_details,
|
||||
request_data,
|
||||
_model_call_details.get("current_attempt_api_call_start_time", _first_handoff) is not None,
|
||||
_billed,
|
||||
_deployment_id,
|
||||
_base_model,
|
||||
dispatched_call_type,
|
||||
)
|
||||
if Logging.usage_capture_callback is not None
|
||||
and (_billed or _model_call_details.get("combined_usage_object") is not None)
|
||||
else None
|
||||
)
|
||||
_current_observation: Final = (
|
||||
_recorded_observation
|
||||
if isinstance(_recorded_observation, UsageObservation)
|
||||
else UsageObservation.UNKNOWN
|
||||
if _current_usage is None
|
||||
else UsageObservation.KNOWN
|
||||
if _model_call_details.get("combined_usage_object") is not None
|
||||
else UsageObservation.ESTIMATED
|
||||
)
|
||||
_capture_observation: Final = UsageCaptureObservation.from_usage(
|
||||
_current_usage[0] if _current_usage is not None and isinstance(_current_usage[0], Usage) else None,
|
||||
_current_observation,
|
||||
float(_current_usage[1])
|
||||
if _current_usage is not None and isinstance(_current_usage[1], (int, float))
|
||||
else None,
|
||||
)
|
||||
_entries: Final = (
|
||||
("start_time", _model_call_details.get("start_time")),
|
||||
("first_api_call_start_time", _first_handoff),
|
||||
|
|
@ -1747,6 +1785,16 @@ def _failure_fields_to_lift(
|
|||
("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)),
|
||||
("standard_logging_object", _model_call_details.get("standard_logging_object")),
|
||||
("billed_failed_attempts", _earlier_attempts or None),
|
||||
("_accounting_capture", _model_call_details.get("_accounting_capture")),
|
||||
("_accounting_pricing_snapshot", _model_call_details.get("_accounting_pricing_snapshot")),
|
||||
("_accounting_finalized_cost", _logging_obj.finalized_cost if isinstance(_logging_obj, Logging) else None),
|
||||
(
|
||||
"_accounting_allocation",
|
||||
_logging_obj.cost_allowance_allocation if isinstance(_logging_obj, Logging) else None,
|
||||
),
|
||||
("_accounting_usage_observed", _current_observation if Logging.usage_capture_callback is not None else None),
|
||||
("_accounting_usage_snapshot", _capture_observation if Logging.usage_capture_callback is not None else None),
|
||||
("_accounting_prior_captures", _model_call_details.get("_accounting_prior_captures")),
|
||||
)
|
||||
return MappingProxyType({key: value for key, value in _entries if value is not None})
|
||||
|
||||
|
|
|
|||
|
|
@ -223,6 +223,10 @@ async def _execute_query_pipeline(
|
|||
"""
|
||||
Execute the RAG query pipeline.
|
||||
"""
|
||||
parent_request_id: Final[str | None] = TypeAdapter[str | None](str | None).validate_python(
|
||||
TypeAdapter(dict[str, object]).validate_python(kwargs).get("litellm_call_id")
|
||||
)
|
||||
|
||||
# Extract router from kwargs - use it for completion if available
|
||||
# to properly resolve virtual model names
|
||||
router: Final[Router | None] = kwargs.pop("router", None)
|
||||
|
|
@ -252,7 +256,7 @@ async def _execute_query_pipeline(
|
|||
forwarded_search_params: Final = MappingProxyType(
|
||||
{**provider_search_params, **kwargs, **filter_search_params, **store_search_params}
|
||||
)
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(parent_request_id):
|
||||
search_response: Final = await litellm.vector_stores.asearch(
|
||||
vector_store_id=retrieval_config["vector_store_id"],
|
||||
query=query_text,
|
||||
|
|
@ -286,7 +290,7 @@ async def _execute_query_pipeline(
|
|||
if rerank and rerank.get("enabled"):
|
||||
documents: Final = RAGQuery.extract_documents_from_search(search_response)
|
||||
if documents:
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(parent_request_id):
|
||||
rerank_response = await litellm.arerank(
|
||||
model=rerank["model"],
|
||||
query=query_text,
|
||||
|
|
@ -304,7 +308,7 @@ async def _execute_query_pipeline(
|
|||
modified_messages: Final = messages[:-1] + [context_message] + [messages[-1]]
|
||||
|
||||
# Use router if available to properly resolve virtual model names
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(parent_request_id):
|
||||
if router is not None:
|
||||
response = await router.acompletion(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
|
||||
from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
|
||||
|
|
@ -148,6 +149,9 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=bridged_request,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
||||
|
|
@ -240,5 +244,8 @@ class LiteLLMCompletionTransformationHandler:
|
|||
responses_api_request=responses_api_request,
|
||||
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
|
||||
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
|
||||
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
|
||||
|
|
|
|||
|
|
@ -200,6 +200,7 @@ class ResponsesSessionHandler:
|
|||
previous_response_id: str,
|
||||
item_cache_control: bool = False,
|
||||
client_response_ids: tuple[str, ...] = (),
|
||||
tool_call_provider_fields: bool = False,
|
||||
stored_turns: StoredTurns | None = None,
|
||||
) -> ChatCompletionSession:
|
||||
"""
|
||||
|
|
@ -241,9 +242,13 @@ class ResponsesSessionHandler:
|
|||
for index in range(start.index, replayed_turns)
|
||||
],
|
||||
item_cache_control,
|
||||
tool_call_provider_fields,
|
||||
)
|
||||
history_before_replay: Final = ResponsesSessionHandler._history_of_turns(
|
||||
all_spend_logs[: start.index], proxy_server_requests[: start.index], item_cache_control
|
||||
all_spend_logs[: start.index],
|
||||
proxy_server_requests[: start.index],
|
||||
item_cache_control,
|
||||
tool_call_provider_fields,
|
||||
)
|
||||
chat_completion_message_history: Final = (
|
||||
list(with_stored_calls_the_input_answers(replayed_history, history_before_replay))
|
||||
|
|
@ -305,6 +310,7 @@ class ResponsesSessionHandler:
|
|||
spend_logs: Sequence["SpendLogsPayload"],
|
||||
proxy_server_requests: Sequence[Mapping[str, object] | None],
|
||||
item_cache_control: bool,
|
||||
tool_call_provider_fields: bool,
|
||||
) -> list["SessionMessage"]:
|
||||
return list(
|
||||
chain.from_iterable(
|
||||
|
|
@ -313,6 +319,7 @@ class ResponsesSessionHandler:
|
|||
proxy_server_request_dict=proxy_server_request,
|
||||
chat_completion_message_history=[],
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
)
|
||||
for spend_log, proxy_server_request in zip(spend_logs, proxy_server_requests, strict=True)
|
||||
)
|
||||
|
|
@ -330,6 +337,7 @@ class ResponsesSessionHandler:
|
|||
| Message
|
||||
],
|
||||
item_cache_control: bool = False,
|
||||
tool_call_provider_fields: bool = False,
|
||||
):
|
||||
"""
|
||||
Extend the chat completion message history with the spend log payload
|
||||
|
|
@ -362,6 +370,7 @@ class ResponsesSessionHandler:
|
|||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
only_since_compaction=True,
|
||||
)
|
||||
)
|
||||
|
|
@ -377,6 +386,7 @@ class ResponsesSessionHandler:
|
|||
responses_api_request={},
|
||||
replay_reasoning=True,
|
||||
item_cache_control=item_cache_control,
|
||||
tool_call_provider_fields=tool_call_provider_fields,
|
||||
only_since_compaction=True,
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import litellm
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
|
||||
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
|
||||
from litellm.llms.anthropic.messages.native.utils import openai_chat_refusal_text
|
||||
from litellm.main import stream_chunk_builder
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
|
||||
from litellm.responses.litellm_completion_transformation.custom_tools import (
|
||||
|
|
@ -35,6 +36,7 @@ from litellm.types.llms.openai import (
|
|||
ContentPartAddedEvent,
|
||||
ContentPartDoneEvent,
|
||||
ContentPartDonePartOutputText,
|
||||
ContentPartDonePartRefusal,
|
||||
FunctionCallArgumentsDeltaEvent,
|
||||
FunctionCallArgumentsDoneEvent,
|
||||
OutputItemAddedEvent,
|
||||
|
|
@ -45,6 +47,8 @@ from litellm.types.llms.openai import (
|
|||
ReasoningSummaryPartDoneEvent,
|
||||
ReasoningSummaryTextDeltaEvent,
|
||||
ReasoningSummaryTextDoneEvent,
|
||||
RefusalDeltaEvent,
|
||||
RefusalDoneEvent,
|
||||
ResponseCompletedEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseIncompleteEvent,
|
||||
|
|
@ -88,6 +92,20 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str |
|
|||
)
|
||||
|
||||
|
||||
def _tool_call_objects(tool_calls: object) -> tuple[object, ...]:
|
||||
return tuple(tool_calls) if isinstance(tool_calls, list) else ()
|
||||
|
||||
|
||||
def _tool_call_provider_specific_fields(tool_call: object) -> Mapping[str, object]:
|
||||
mapping: Final = as_str_mapping(tool_call)
|
||||
fields: Final[object] = (
|
||||
mapping.get("provider_specific_fields")
|
||||
if mapping is not None
|
||||
else getattr(tool_call, "provider_specific_fields", None)
|
||||
)
|
||||
return {"provider_specific_fields": fields} if fields else {}
|
||||
|
||||
|
||||
def _is_signed_thinking_block(block: object) -> bool:
|
||||
fields: Final = as_str_mapping(block)
|
||||
return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data"))
|
||||
|
|
@ -110,6 +128,7 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool:
|
|||
delta.function_call,
|
||||
getattr(delta, "images", None),
|
||||
getattr(delta, "audio", None),
|
||||
openai_chat_refusal_text(delta),
|
||||
)
|
||||
return any(fields) or _delta_has_signed_thinking_block(delta)
|
||||
|
||||
|
|
@ -119,6 +138,13 @@ def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool:
|
|||
return has_reasoning and bool(delta.content or delta.tool_calls)
|
||||
|
||||
|
||||
_WEB_SEARCH_PROGRESS_EVENTS: Final = (
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
|
||||
)
|
||||
|
||||
|
||||
class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
||||
"""
|
||||
Async iterator for processing streaming responses from the Responses API.
|
||||
|
|
@ -132,6 +158,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
responses_api_request: ResponsesAPIOptionalRequestParams,
|
||||
custom_llm_provider: str | None = None,
|
||||
litellm_metadata: dict | None = None,
|
||||
tool_call_provider_fields: bool = False,
|
||||
):
|
||||
self.model: str = model
|
||||
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper
|
||||
|
|
@ -139,6 +166,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request
|
||||
self.custom_llm_provider: str | None = custom_llm_provider
|
||||
self.litellm_metadata: dict | None = litellm_metadata or {}
|
||||
self.tool_call_provider_fields: Final = tool_call_provider_fields
|
||||
# Store lightweight dict snapshots for stream_chunk_builder to reduce
|
||||
# repeated Pydantic attribute access in end-of-stream assembly.
|
||||
self.collected_chat_completion_chunks: list[dict[str, object]] = []
|
||||
|
|
@ -152,6 +180,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
self.sent_output_text_done_event: bool = False
|
||||
self.sent_output_content_part_done_event: bool = False
|
||||
self.sent_output_item_done_event: bool = False
|
||||
self._sent_refusal_events: bool = False
|
||||
self._annotation_index: int = 0
|
||||
self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = ()
|
||||
self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
|
||||
|
|
@ -490,10 +519,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
except Exception:
|
||||
tool_calls = None
|
||||
|
||||
if not tool_calls or not isinstance(tool_calls, list):
|
||||
return
|
||||
|
||||
for tc in tool_calls:
|
||||
for tc in _tool_call_objects(tool_calls):
|
||||
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
|
||||
if not call_id_raw:
|
||||
continue
|
||||
|
|
@ -568,7 +594,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=output_index,
|
||||
sequence_number=self._sequence_number,
|
||||
item=BaseLiteLLMOpenAIResponseObject(**item_kwargs),
|
||||
item=BaseLiteLLMOpenAIResponseObject(
|
||||
**item_kwargs,
|
||||
**(_tool_call_provider_specific_fields(tc) if self.tool_call_provider_fields else {}),
|
||||
),
|
||||
)
|
||||
self._pending_tool_events.append(item_done_event)
|
||||
|
||||
|
|
@ -592,11 +621,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
}
|
||||
),
|
||||
)
|
||||
for event_type, event_class in (
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
|
||||
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
|
||||
):
|
||||
progress_events: Final = (
|
||||
_WEB_SEARCH_PROGRESS_EVENTS[:1] if item.status == "in_progress" else _WEB_SEARCH_PROGRESS_EVENTS
|
||||
)
|
||||
for event_type, event_class in progress_events:
|
||||
self._sequence_number += 1
|
||||
event = event_class(type=event_type, output_index=output_index, item_id=item.id)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
|
|
@ -798,9 +826,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return event
|
||||
|
||||
def _queue_message_item_added_events(self) -> None:
|
||||
if not self.sent_message_item_added_event:
|
||||
self._queue_message_item_added_event()
|
||||
self.sent_content_part_added_event = True
|
||||
self._pending_response_events.append(self.create_content_part_added_event())
|
||||
|
||||
def _queue_message_item_added_event(self) -> None:
|
||||
item_id: Final = self._message_item_id()
|
||||
self.sent_message_item_added_event = True
|
||||
self.sent_content_part_added_event = True
|
||||
self._sequence_number += 1
|
||||
event: Final = OutputItemAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
|
||||
|
|
@ -817,7 +850,47 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
)
|
||||
event.__dict__["sequence_number"] = self._sequence_number
|
||||
self._pending_response_events.append(event)
|
||||
self._pending_response_events.append(self.create_content_part_added_event())
|
||||
|
||||
def _queue_refusal_events(self, refusal: str) -> None:
|
||||
item_id: Final = self._message_item_id()
|
||||
output_index: Final = self._message_index()
|
||||
content_index: Final = 1 if self.sent_content_part_added_event else 0
|
||||
events: Final = (
|
||||
ContentPartAddedEvent(
|
||||
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=ContentPartDonePartRefusal(type="refusal", refusal=""),
|
||||
),
|
||||
RefusalDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.REFUSAL_DELTA,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
delta=refusal,
|
||||
),
|
||||
RefusalDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.REFUSAL_DONE,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
refusal=refusal,
|
||||
),
|
||||
ContentPartDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
|
||||
item_id=item_id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=ContentPartDonePartRefusal(type="refusal", refusal=refusal),
|
||||
),
|
||||
)
|
||||
first_sequence_number: Final = self._sequence_number + 1
|
||||
self._sequence_number += len(events)
|
||||
self._pending_response_events.extend(
|
||||
event.model_copy(update={"sequence_number": sequence_number})
|
||||
for sequence_number, event in enumerate(events, start=first_sequence_number)
|
||||
)
|
||||
|
||||
def _merge_provider_specific_fields(self, src: dict) -> None:
|
||||
"""Merge provider_specific_fields using last-value-wins for lists.
|
||||
|
|
@ -974,14 +1047,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
if self._cached_item_id is None:
|
||||
self._cached_item_id = f"msg_{uuid.uuid4()}"
|
||||
|
||||
text: Final = litellm_complete_object.choices[0].message.content or ""
|
||||
annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None)
|
||||
message: Final = litellm_complete_object.choices[0].message
|
||||
text: Final = message.content or ""
|
||||
annotations = getattr(message, "annotations", None)
|
||||
|
||||
response_annotations: Final = (
|
||||
LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
|
||||
annotations=annotations
|
||||
)
|
||||
)
|
||||
text_parts: Final = (
|
||||
({"type": "output_text", "text": text, "annotations": response_annotations},)
|
||||
if self.sent_content_part_added_event
|
||||
else ()
|
||||
)
|
||||
refusal: Final = openai_chat_refusal_text(message)
|
||||
refusal_parts: Final = ({"type": "refusal", "refusal": refusal},) if refusal is not None else ()
|
||||
return OutputItemDoneEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
|
||||
output_index=self._message_index(),
|
||||
|
|
@ -992,13 +1073,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
"status": "completed",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": text,
|
||||
"annotations": response_annotations,
|
||||
}
|
||||
],
|
||||
"content": [*text_parts, *refusal_parts],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
|
@ -1050,22 +1125,33 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
def return_default_done_events(
|
||||
self, litellm_complete_object: ModelResponse
|
||||
) -> BaseLiteLLMOpenAIResponseObject | None:
|
||||
message: Final = litellm_complete_object.choices[0].message
|
||||
refusal: Final = openai_chat_refusal_text(message)
|
||||
if self.sent_message_item_added_event is False:
|
||||
final_content: Final = litellm_complete_object.choices[0].message.content or ""
|
||||
if not final_content:
|
||||
final_content: Final = message.content or ""
|
||||
if not final_content and refusal is None:
|
||||
self.sent_output_text_done_event = True
|
||||
self.sent_output_content_part_done_event = True
|
||||
self.sent_output_item_done_event = True
|
||||
return None
|
||||
self._queue_message_item_added_events()
|
||||
if final_content:
|
||||
self._queue_message_item_added_events()
|
||||
else:
|
||||
self._queue_message_item_added_event()
|
||||
return self._pending_response_events.pop(0)
|
||||
if self.sent_output_text_done_event is False:
|
||||
if self.sent_output_text_done_event is False and self.sent_content_part_added_event:
|
||||
self.sent_output_text_done_event = True
|
||||
return self.create_output_text_done_event(litellm_complete_object)
|
||||
if self.sent_output_content_part_done_event is False:
|
||||
if self.sent_output_content_part_done_event is False and self.sent_content_part_added_event:
|
||||
self.sent_output_content_part_done_event = True
|
||||
return self.create_output_content_part_done_event(litellm_complete_object)
|
||||
if refusal is not None and not self._sent_refusal_events:
|
||||
self._sent_refusal_events = True
|
||||
self._queue_refusal_events(refusal)
|
||||
return self._pending_response_events.pop(0)
|
||||
if self.sent_output_item_done_event is False:
|
||||
self.sent_output_text_done_event = True
|
||||
self.sent_output_content_part_done_event = True
|
||||
self.sent_output_item_done_event = True
|
||||
return self.create_output_item_done_event(litellm_complete_object)
|
||||
return None
|
||||
|
|
@ -1169,7 +1255,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
# DO NOT create message item
|
||||
return
|
||||
|
||||
# Default: message
|
||||
if openai_chat_refusal_text(delta) is not None and not delta.content:
|
||||
self._queue_message_item_added_event()
|
||||
return
|
||||
self._queue_message_item_added_events()
|
||||
return
|
||||
|
||||
|
|
@ -1413,7 +1501,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
# Priority 2: Handle text deltas
|
||||
delta_content: Final = self._get_delta_string_from_streaming_choices(chunk.choices)
|
||||
if delta_content:
|
||||
if not self.sent_message_item_added_event:
|
||||
if not self.sent_content_part_added_event:
|
||||
self._queue_message_item_added_events()
|
||||
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
|
||||
self._sequence_number += 1
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import uuid
|
|||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import chain, groupby
|
||||
from itertools import accumulate, chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -28,7 +28,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
|
|||
from openai.types.chat.chat_completion_named_tool_choice_param import (
|
||||
Function as NamedToolChoiceFunction,
|
||||
)
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch, ResponseOutputRefusal
|
||||
from openai.types.responses.response_compaction_item import ResponseCompactionItem
|
||||
from openai.types.responses.response_create_params import ResponseInputParam
|
||||
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
|
||||
|
|
@ -54,7 +54,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
whole_messages_within_cache_control_limit,
|
||||
with_cache_control,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
)
|
||||
|
|
@ -64,7 +64,9 @@ from litellm.litellm_core_utils.hidden_params import (
|
|||
set_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
|
||||
from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text
|
||||
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
|
||||
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
|
||||
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
|
||||
|
|
@ -373,8 +375,9 @@ class _ResponsesToolCallItem(Protocol):
|
|||
def get(self, key: str, /) -> object: ...
|
||||
|
||||
|
||||
class _MarkedToolCallChunk(ChatCompletionToolCallChunk):
|
||||
class _ReplayedToolCallChunk(ChatCompletionToolCallChunk, total=False):
|
||||
cache_control: ReadOnly[ChatCompletionCachedContent]
|
||||
provider_specific_fields: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
class _MarkedToolMessage(ChatCompletionToolMessage):
|
||||
|
|
@ -446,6 +449,32 @@ def _input_item_role(input_item: Mapping[str, object]) -> str:
|
|||
return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated
|
||||
|
||||
|
||||
def _starts_a_turn(input_item: Mapping[str, object]) -> bool:
|
||||
item_type: Final = input_item.get("type")
|
||||
if item_type in TOOL_CALL_OUTPUT_ITEM_TYPES:
|
||||
return True
|
||||
return item_type in (None, "message") and input_item.get("role") != "assistant"
|
||||
|
||||
|
||||
def _is_unrun_hosted_search(input_item: Mapping[str, object]) -> bool:
|
||||
return input_item.get("type") == "web_search_call" and input_item.get("status") == "in_progress"
|
||||
|
||||
|
||||
def _without_encrypted_content(input_item: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return {key: value for key, value in input_item.items() if key != "encrypted_content"}
|
||||
|
||||
|
||||
def _without_paused_turn_thinking(input_items: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
|
||||
turns: Final = tuple(accumulate(int(_starts_a_turn(item)) for item in input_items))
|
||||
paused_turns: Final = frozenset(
|
||||
turn for turn, item in zip(turns, input_items, strict=True) if _is_unrun_hosted_search(item)
|
||||
)
|
||||
return tuple(
|
||||
_without_encrypted_content(item) if turn in paused_turns and item.get("type") == "reasoning" else item
|
||||
for turn, item in zip(turns, input_items, strict=True)
|
||||
)
|
||||
|
||||
|
||||
SessionMessage: TypeAlias = (
|
||||
AllMessageValues
|
||||
| GenericChatCompletionMessage
|
||||
|
|
@ -611,17 +640,28 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
|
|||
return None
|
||||
|
||||
|
||||
def _without_cache_control_key(item: object) -> object:
|
||||
def _without_keys(item: object, keys: frozenset[str]) -> object:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"}
|
||||
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key not in keys}
|
||||
|
||||
|
||||
def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if not isinstance(input, list):
|
||||
def _without_item_keys(input: str | ResponseInputParam, keys: frozenset[str]) -> str | ResponseInputParam:
|
||||
if not keys or not isinstance(input, list):
|
||||
return input
|
||||
return cast( # cast-ok: the same input items, each minus its own cache_control key
|
||||
ResponseInputParam, [_without_cache_control_key(item) for item in input]
|
||||
return cast( # cast-ok: the same input items, each minus the given keys
|
||||
ResponseInputParam, [_without_keys(item, keys) for item in input]
|
||||
)
|
||||
|
||||
|
||||
def _dropped_item_keys(item_cache_control: bool, tool_call_provider_fields: bool) -> frozenset[str]:
|
||||
return frozenset(
|
||||
key
|
||||
for key, kept in (
|
||||
("cache_control", item_cache_control),
|
||||
("provider_specific_fields", tool_call_provider_fields),
|
||||
)
|
||||
if not kept
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1114,6 +1154,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
responses_api_request=responses_api_request,
|
||||
replay_reasoning=True,
|
||||
item_cache_control=keeps_item_marks,
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
|
||||
native_compaction=native_compaction,
|
||||
only_since_compaction=not native_compaction,
|
||||
)
|
||||
|
|
@ -1177,6 +1218,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
responses_api_request: ResponsesAPIOptionalRequestParams | dict,
|
||||
replay_reasoning: bool = False,
|
||||
item_cache_control: bool = False,
|
||||
tool_call_provider_fields: bool = False,
|
||||
native_compaction: bool = False,
|
||||
only_since_compaction: bool = False,
|
||||
) -> list[
|
||||
|
|
@ -1194,7 +1236,9 @@ class LiteLLMCompletionResponsesConfig:
|
|||
assistant ``reasoning_content`` and signed ``thinking_blocks`` so the
|
||||
provider gets its own chain-of-thought back instead of reading it as
|
||||
visible text. ``item_cache_control`` belongs to the same callers, when
|
||||
the target keeps an input item's ``cache_control`` on the message.
|
||||
the target keeps an input item's ``cache_control`` on the message, and
|
||||
``tool_call_provider_fields`` when the target reads a function call's
|
||||
thought signature from ``provider_specific_fields`` instead of its id.
|
||||
|
||||
Callers that only inspect the messages (token counting, rate limiting,
|
||||
guardrail scanning) leave both off, because they need every piece of text
|
||||
|
|
@ -1232,7 +1276,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction)
|
||||
messages.extend(
|
||||
LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
|
||||
input=read_input if item_cache_control else without_item_cache_control(read_input),
|
||||
input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)),
|
||||
replay_reasoning=replay_reasoning,
|
||||
native_compaction=native_compaction,
|
||||
)
|
||||
|
|
@ -1290,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id,
|
||||
item_cache_control=keeps_item_marks,
|
||||
client_response_ids=previous_response_client_ids,
|
||||
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
|
||||
stored_turns=stored_turns,
|
||||
)
|
||||
chat_completion_session: Final = (
|
||||
|
|
@ -1416,7 +1461,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
messages.append(ChatCompletionUserMessage(role="user", content=input))
|
||||
elif isinstance(input, list):
|
||||
existing_tool_call_ids: Final[set[str]] = set()
|
||||
for _input in input:
|
||||
for _input in _without_paused_turn_thinking(input):
|
||||
chat_completion_messages = (
|
||||
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
|
||||
input_item=_input,
|
||||
|
|
@ -2473,7 +2518,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
tool_call: Final = (
|
||||
ChatCompletionToolCallChunk(id=tool_call_id, type="function", function=tool_call_function, index=0)
|
||||
if function_call_cache_control is None
|
||||
else _MarkedToolCallChunk(
|
||||
else _ReplayedToolCallChunk(
|
||||
id=tool_call_id,
|
||||
type="function",
|
||||
function=tool_call_function,
|
||||
|
|
@ -2481,9 +2526,17 @@ class LiteLLMCompletionResponsesConfig:
|
|||
cache_control=function_call_cache_control,
|
||||
)
|
||||
)
|
||||
provider_specific_fields: Final = as_str_mapping(function_call.get("provider_specific_fields"))
|
||||
replayed_tool_call: Final = (
|
||||
tool_call
|
||||
if provider_specific_fields is None
|
||||
else cast( # cast-ok: the same tool call plus the provider_specific_fields the client sent back
|
||||
_ReplayedToolCallChunk, {**tool_call, "provider_specific_fields": provider_specific_fields}
|
||||
)
|
||||
)
|
||||
|
||||
# Create an assistant message with the tool call
|
||||
return [ChatCompletionResponseMessage(tool_calls=[tool_call], role="assistant", content=None)]
|
||||
return [ChatCompletionResponseMessage(tool_calls=[replayed_tool_call], role="assistant", content=None)]
|
||||
|
||||
@staticmethod
|
||||
def _transform_input_file_item_to_file_item(item: Mapping[str, object]) -> _ResponsesFilePart:
|
||||
|
|
@ -2528,45 +2581,55 @@ class LiteLLMCompletionResponsesConfig:
|
|||
for item in content:
|
||||
if isinstance(item, str):
|
||||
content_list.append(item)
|
||||
elif isinstance(item, dict):
|
||||
if (
|
||||
media := decode_responses_media(
|
||||
fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
|
||||
elif (
|
||||
isinstance(item, dict)
|
||||
and (
|
||||
part
|
||||
:= LiteLLMCompletionResponsesConfig._transform_responses_api_content_part_to_chat_completion_part(
|
||||
_validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
|
||||
)
|
||||
) is not None:
|
||||
content_list.append(dict(media))
|
||||
elif fields.get("type") == "input_image":
|
||||
content_list.append(dict(_responses_image_part(fields)))
|
||||
elif fields.get("type") == "input_file":
|
||||
content_list.append(dict(_responses_file_part(fields)))
|
||||
elif fields.get("type") == "input_audio":
|
||||
content_list.append(dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields)))
|
||||
elif fields.get("type") == "encrypted_content":
|
||||
encrypted_content = fields.get("encrypted_content")
|
||||
if encrypted_content is not None:
|
||||
content_list.append(
|
||||
OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content))
|
||||
)
|
||||
else:
|
||||
# Skip text blocks with None text to avoid downstream errors
|
||||
text_value = fields.get("text")
|
||||
if text_value is None:
|
||||
continue
|
||||
content_block: dict[str, object] = {
|
||||
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
|
||||
content_type
|
||||
if isinstance(content_type := fields.get("type"), str) and content_type
|
||||
else "text"
|
||||
),
|
||||
"text": text_value,
|
||||
}
|
||||
if "cache_control" in fields:
|
||||
content_block["cache_control"] = fields["cache_control"]
|
||||
content_list.append(content_block)
|
||||
)
|
||||
is not None
|
||||
):
|
||||
content_list.append(part)
|
||||
return content_list
|
||||
else:
|
||||
raise ValueError(f"Invalid content type: {type(content)}")
|
||||
|
||||
@staticmethod
|
||||
def _transform_responses_api_content_part_to_chat_completion_part(
|
||||
fields: dict[str, object],
|
||||
) -> dict[str, object] | None:
|
||||
if (media := decode_responses_media(fields)) is not None:
|
||||
return dict(media)
|
||||
part_type: Final = fields.get("type")
|
||||
if part_type == "input_image":
|
||||
return dict(_responses_image_part(fields))
|
||||
if part_type == "input_file":
|
||||
return dict(_responses_file_part(fields))
|
||||
if part_type == "input_audio":
|
||||
return dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields))
|
||||
if part_type == "refusal":
|
||||
refusal: Final = fields.get("refusal")
|
||||
return dict(OpenAIChatCompletionTextObject(type="text", text=refusal)) if isinstance(refusal, str) else None
|
||||
if part_type == "encrypted_content":
|
||||
encrypted_content: Final = fields.get("encrypted_content")
|
||||
return (
|
||||
None
|
||||
if encrypted_content is None
|
||||
else dict(OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content)))
|
||||
)
|
||||
text_value: Final = fields.get("text")
|
||||
if text_value is None:
|
||||
return None
|
||||
return {
|
||||
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
|
||||
part_type if isinstance(part_type, str) and part_type else "text"
|
||||
),
|
||||
"text": text_value,
|
||||
**({"cache_control": fields["cache_control"]} if "cache_control" in fields else {}),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _get_chat_completion_request_content_type(
|
||||
content_type: str,
|
||||
|
|
@ -3045,6 +3108,18 @@ class LiteLLMCompletionResponsesConfig:
|
|||
# Default to completed for unknown finish reasons
|
||||
return "completed"
|
||||
|
||||
@staticmethod
|
||||
def _is_paused_turn(choice: Choices) -> bool:
|
||||
return anthropic_stop_reason(choice.message) == "pause_turn"
|
||||
|
||||
@staticmethod
|
||||
def _responses_status_for_choice(choice: Choices) -> ResponsesAPIStatus:
|
||||
if LiteLLMCompletionResponsesConfig._is_paused_turn(choice):
|
||||
return "completed"
|
||||
return LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
choice.finish_reason
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _incomplete_details_for_finish_reason(
|
||||
finish_reason: str | None,
|
||||
|
|
@ -3181,19 +3256,21 @@ class LiteLLMCompletionResponsesConfig:
|
|||
"""
|
||||
if isinstance(chat_completion_response, dict):
|
||||
chat_completion_response = ModelResponse(**chat_completion_response)
|
||||
# Get finish_reason from the first choice to determine overall status
|
||||
finish_reason: str | None = None
|
||||
choices: Final[list[Choices]] = getattr(chat_completion_response, "choices", [])
|
||||
if choices and len(choices) > 0:
|
||||
finish_reason = choices[0].finish_reason
|
||||
finish_reason: Final = choices[0].finish_reason if choices else None
|
||||
paused_turn: Final = bool(choices) and LiteLLMCompletionResponsesConfig._is_paused_turn(choices[0])
|
||||
|
||||
echoed: Final = _echoable_request_params(responses_api_request)
|
||||
chat_usage: Final = getattr(chat_completion_response, "usage", None)
|
||||
incomplete_details: Final = LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
|
||||
finish_reason=finish_reason,
|
||||
existing=getattr(chat_completion_response, "incomplete_details", None),
|
||||
completion_tokens=getattr(chat_usage, "completion_tokens", None),
|
||||
max_output_tokens=echoed.get("max_output_tokens"),
|
||||
incomplete_details: Final = (
|
||||
None
|
||||
if paused_turn
|
||||
else LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
|
||||
finish_reason=finish_reason,
|
||||
existing=getattr(chat_completion_response, "incomplete_details", None),
|
||||
completion_tokens=getattr(chat_usage, "completion_tokens", None),
|
||||
max_output_tokens=echoed.get("max_output_tokens"),
|
||||
)
|
||||
)
|
||||
|
||||
responses_api_response: Final[ResponsesAPIResponse] = ResponsesAPIResponse(
|
||||
|
|
@ -3220,8 +3297,8 @@ class LiteLLMCompletionResponsesConfig:
|
|||
max_output_tokens=echoed.get("max_output_tokens"),
|
||||
previous_response_id=echoed.get("previous_response_id"),
|
||||
reasoning=echoed.get("reasoning"),
|
||||
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
finish_reason
|
||||
status=(
|
||||
LiteLLMCompletionResponsesConfig._responses_status_for_choice(choices[0]) if choices else "incomplete"
|
||||
),
|
||||
text=echoed.get("text") or {},
|
||||
truncation=echoed.get("truncation"),
|
||||
|
|
@ -3367,9 +3444,7 @@ class LiteLLMCompletionResponsesConfig:
|
|||
GenericResponseOutputItem(
|
||||
type="reasoning",
|
||||
id=mint_reasoning_item_id(),
|
||||
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
choice.finish_reason
|
||||
),
|
||||
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
|
||||
role="assistant",
|
||||
content=[
|
||||
OutputText(
|
||||
|
|
@ -3492,24 +3567,34 @@ class LiteLLMCompletionResponsesConfig:
|
|||
choice=choice,
|
||||
)
|
||||
message_output_items.extend(image_generation_items)
|
||||
elif choice.message.content is not None or getattr(choice.message, "annotations", None):
|
||||
elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message):
|
||||
message_output_items.append(
|
||||
GenericResponseOutputItem(
|
||||
type="message",
|
||||
id=f"msg_{uuid.uuid4()}",
|
||||
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
|
||||
choice.finish_reason
|
||||
),
|
||||
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
|
||||
role=choice.message.role,
|
||||
content=[
|
||||
LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(
|
||||
choice.message
|
||||
)
|
||||
],
|
||||
content=list(content),
|
||||
)
|
||||
)
|
||||
return message_output_items
|
||||
|
||||
@staticmethod
|
||||
def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]:
|
||||
refusal: Final = openai_chat_refusal_text(message)
|
||||
refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else ()
|
||||
has_text_part: Final = (
|
||||
bool(message.content)
|
||||
or bool(getattr(message, "annotations", None))
|
||||
or (message.content is not None and not refusal_parts)
|
||||
)
|
||||
text_parts: Final = (
|
||||
(LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),)
|
||||
if has_text_part
|
||||
else ()
|
||||
)
|
||||
return (*text_parts, *refusal_parts)
|
||||
|
||||
@staticmethod
|
||||
def _transform_responses_api_outputs_to_chat_completion_messages(
|
||||
responses_api_output: ResponsesAPIResponse,
|
||||
|
|
|
|||
|
|
@ -437,7 +437,7 @@ async def aresponses_api_with_mcp(
|
|||
#########################################################
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.carry_failed_attempt_usage()
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
|
||||
response: Final = await aresponses(
|
||||
input=input,
|
||||
model=model,
|
||||
|
|
@ -514,7 +514,9 @@ async def aresponses_api_with_mcp(
|
|||
tool_calls=tool_calls, tool_results=tool_results
|
||||
)
|
||||
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(
|
||||
logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
|
||||
):
|
||||
final_response = await LiteLLM_Proxy_MCP_Handler.make_follow_up_call(
|
||||
follow_up_input=follow_up_input,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -663,7 +663,7 @@ async def acompletion_with_mcp(
|
|||
logging_obj: Final = kwargs.get("litellm_logging_obj")
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.carry_failed_attempt_usage()
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
|
||||
initial_response: Final = await litellm_acompletion(**initial_call_args)
|
||||
first_round_breakdown: Final = logging_obj.cost_breakdown if isinstance(logging_obj, LiteLLMLoggingObj) else None
|
||||
|
||||
|
|
@ -713,7 +713,7 @@ async def acompletion_with_mcp(
|
|||
follow_up_call_args["messages"] = follow_up_messages
|
||||
follow_up_call_args["stream"] = stream
|
||||
|
||||
with internal_sub_call():
|
||||
with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
|
||||
final_response: Final = await litellm_acompletion(**follow_up_call_args)
|
||||
|
||||
if not isinstance(final_response, ModelResponse):
|
||||
|
|
|
|||
|
|
@ -35,11 +35,13 @@ import litellm
|
|||
from litellm.constants import (
|
||||
EMPTY_MAPPING,
|
||||
LITELLM_MAX_STREAMING_DURATION_SECONDS,
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
|
||||
STREAM_SSE_DONE_STRING,
|
||||
)
|
||||
from litellm.exceptions import MidStreamFallbackError, RateLimitError
|
||||
from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
UPSTREAM_USAGE_COST,
|
||||
process_response_headers,
|
||||
provider_reported_usage_cost,
|
||||
set_response_cost_in_hidden_params,
|
||||
|
|
@ -100,6 +102,12 @@ class ProjectQuotaCallback(Protocol):
|
|||
) -> None: ...
|
||||
|
||||
|
||||
class TokenLimitCallback(Protocol):
|
||||
async def enforce_token_limits_for_frame(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
|
||||
) -> None: ...
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_openai_response_types():
|
||||
from litellm.types.llms import openai as openai_types
|
||||
|
|
@ -177,6 +185,7 @@ def _typed_gets_litellm_params(fn: _GetsLitellmParams) -> _GetsLitellmParams:
|
|||
_SHOULD_STORE_RESULT_IN_CACHE_ATTR: Final = "_should_store_result_in_cache"
|
||||
_UNMASK_PII_TEXT_ATTR: Final = "_unmask_pii_text"
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
_HIDDEN_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _load_json_object(payload: str | bytes) -> dict[str, object]:
|
||||
|
|
@ -224,6 +233,11 @@ def spawn_logging_task(
|
|||
|
||||
|
||||
async def drain_logging_tasks(timeout: float = 5.0) -> None:
|
||||
# A task that finished as its loop closed never ran its discard callback, and a
|
||||
# task from another loop cannot be awaited here: forget both, wait on the rest.
|
||||
running_loop: Final = asyncio.get_running_loop()
|
||||
stale: Final = {task for task in _PENDING_LOGGING_TASKS if task.done() or task.get_loop() is not running_loop}
|
||||
_PENDING_LOGGING_TASKS.difference_update(stale)
|
||||
if not _PENDING_LOGGING_TASKS:
|
||||
return
|
||||
_, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout)
|
||||
|
|
@ -686,7 +700,9 @@ class BaseResponsesAPIStreamingIterator:
|
|||
source_hidden: Final[object] = getattr(
|
||||
getattr(self.completed_response, "response", None), "_hidden_params", None
|
||||
)
|
||||
source: Final[Mapping[str, object]] = source_hidden if isinstance(source_hidden, Mapping) else EMPTY_MAPPING
|
||||
source: Final[Mapping[str, object]] = (
|
||||
_HIDDEN_PARAMS.validate_python(source_hidden) if isinstance(source_hidden, Mapping) else {}
|
||||
)
|
||||
processed: Final[object] = source.get("additional_headers") or self._hidden_params.get("additional_headers")
|
||||
raw: Final[object] = source.get("headers") or self._raw_response_headers
|
||||
headers: Final[Mapping[str, object]] = processed if isinstance(processed, Mapping) else EMPTY_MAPPING
|
||||
|
|
@ -696,6 +712,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
target.hidden_params = {
|
||||
"additional_headers": {**headers},
|
||||
"headers": {**raw_headers},
|
||||
**_stamped_bill_hidden_params(source),
|
||||
**existing,
|
||||
}
|
||||
|
||||
|
|
@ -1742,6 +1759,11 @@ def _dump_json_safely(response: BaseModel) -> str | None:
|
|||
return None
|
||||
|
||||
|
||||
def _stamped_bill_hidden_params(source: Mapping[str, object]) -> Mapping[str, object]:
|
||||
stamped: Final = UPSTREAM_USAGE_COST in source
|
||||
return {key: source[key] for key in (UPSTREAM_USAGE_COST, "response_cost") if stamped and key in source}
|
||||
|
||||
|
||||
def _logging_copy(event: object) -> object:
|
||||
"""Hand logging callbacks a copy, so their usage rewrite (Responses shape to chat shape) never
|
||||
reaches the event the caller is iterating. The round trip through ``model_dump`` sidesteps the
|
||||
|
|
@ -2158,6 +2180,25 @@ class _RefusedTurn:
|
|||
ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]]
|
||||
|
||||
|
||||
def with_token_limit_checks(
|
||||
turn_admission: ResponsesWsTurnAdmission | None,
|
||||
token_limit_callbacks: Sequence[TokenLimitCallback],
|
||||
user_api_key_dict: UserAPIKeyAuth | None,
|
||||
requested_model: str | None,
|
||||
) -> ResponsesWsTurnAdmission | None:
|
||||
if not token_limit_callbacks or user_api_key_dict is None:
|
||||
return turn_admission
|
||||
|
||||
async def admit() -> Mapping[str, object]:
|
||||
for callback in token_limit_callbacks:
|
||||
await callback.enforce_token_limits_for_frame(
|
||||
user_api_key_dict=user_api_key_dict, requested_model=requested_model
|
||||
)
|
||||
return MappingProxyType({}) if turn_admission is None else await turn_admission()
|
||||
|
||||
return admit
|
||||
|
||||
|
||||
async def _admit_frame_turn(
|
||||
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str
|
||||
) -> _AdmittedTurn | _RefusedTurn:
|
||||
|
|
@ -2173,22 +2214,24 @@ async def _admit_frame_turn(
|
|||
return _RefusedTurn(e)
|
||||
|
||||
|
||||
def _turn_refusal_frame(refusal: RateLimitError) -> str:
|
||||
def turn_refusal_frame(refusal: RateLimitError) -> str:
|
||||
return json.dumps(
|
||||
{"type": "error", "status": 429, "error": {"type": "rate_limit_exceeded", "message": str(refusal)}}
|
||||
)
|
||||
|
||||
|
||||
def _turn_admission_error_frame(failure: Exception) -> str:
|
||||
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
|
||||
def _turn_admission_error_frame(failure: Exception, status: int) -> str:
|
||||
return json.dumps({"type": "error", "status": status, "error": {"type": "server_error", "message": str(failure)}})
|
||||
|
||||
|
||||
def _turn_rejection_frame(rejection: Exception) -> str:
|
||||
match rejection:
|
||||
case RateLimitError():
|
||||
return _turn_refusal_frame(rejection)
|
||||
return turn_refusal_frame(rejection)
|
||||
case ServiceUnavailableError():
|
||||
return _turn_admission_error_frame(rejection, rejection.status_code)
|
||||
case _:
|
||||
return _turn_admission_error_frame(rejection)
|
||||
return _turn_admission_error_frame(rejection, 500)
|
||||
|
||||
|
||||
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
|
||||
|
|
@ -3015,7 +3058,7 @@ class ResponsesWebSocketStreaming:
|
|||
|
||||
async def _send_turn_refusal(self, refusal: RateLimitError) -> None:
|
||||
try:
|
||||
await self.websocket.send_text(_turn_refusal_frame(refusal))
|
||||
await self.websocket.send_text(turn_refusal_frame(refusal))
|
||||
except Exception: # noqa: BLE001, S110 # client may already be gone
|
||||
pass
|
||||
|
||||
|
|
@ -3569,7 +3612,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
|
||||
)
|
||||
except RateLimitError as e:
|
||||
await self._send_frame(_turn_refusal_frame(e))
|
||||
await self._send_frame(turn_refusal_frame(e))
|
||||
return
|
||||
guarded: Final = await self._guard_frame(raw_message)
|
||||
if guarded is None:
|
||||
|
|
@ -3613,6 +3656,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
call_kwargs["litellm_metadata"] = {
|
||||
**(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}),
|
||||
**verdict.reservation,
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: self.logging_obj.litellm_call_id,
|
||||
}
|
||||
|
||||
try:
|
||||
|
|
|
|||
17773
litellm/router.py
17773
litellm/router.py
File diff suppressed because it is too large
Load diff
|
|
@ -42,6 +42,8 @@ from dataclasses import dataclass, field
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload
|
||||
|
||||
from litellm.router_config import detached
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Any # noqa: TID251 # only to spell the cores' untyped (Unknown) containers and one Any type
|
||||
|
||||
|
|
@ -176,9 +178,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t
|
|||
Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id``
|
||||
set) only by that team, any other deployment by everyone. Needs no router instance.
|
||||
"""
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
return Router._core_deployment_usable_by_team(deployment, team_id)
|
||||
return PythonCore._core_deployment_usable_by_team(deployment, team_id)
|
||||
|
||||
|
||||
def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str:
|
||||
|
|
@ -200,9 +202,9 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str
|
|||
|
||||
def router_init_args() -> tuple[str, ...]:
|
||||
"""The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance."""
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
return tuple(Router.get_valid_args())
|
||||
return tuple(PythonCore.get_valid_args())
|
||||
|
||||
|
||||
def mark_advisor_orchestration_failure(exception: BaseException) -> None:
|
||||
|
|
@ -266,9 +268,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob
|
|||
|
||||
|
||||
def _json_default_stable_id(obj: object) -> str:
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
return Router._json_default_stable_id(obj)
|
||||
return PythonCore._json_default_stable_id(obj)
|
||||
|
||||
|
||||
class RouterRegistry:
|
||||
|
|
@ -529,10 +531,10 @@ class RouterRegistry:
|
|||
``should_include_deployment``), or, without ``model_name``, whether the team may use it at
|
||||
all (was ``deployment_usable_by_team``).
|
||||
"""
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
if model_name is None:
|
||||
return Router._core_deployment_usable_by_team(deployment, team_id)
|
||||
return PythonCore._core_deployment_usable_by_team(deployment, team_id)
|
||||
return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id)
|
||||
|
||||
@overload
|
||||
|
|
@ -1053,6 +1055,12 @@ SETTING_FIELDS: Final[tuple[str, ...]] = (
|
|||
"cache_responses",
|
||||
)
|
||||
|
||||
# Runtime settings kept as plain router attributes rather than ``settings()`` fields. A write
|
||||
# to one (``router.num_retries = 2``) is a changeset too: ``ChangeSet(assign={...})``.
|
||||
PLAIN_SETTING_ATTRIBUTES: Final[frozenset[str]] = frozenset({"num_retries", "max_fallbacks", "retry_after", "timeout"})
|
||||
# Former attributes whose value lives under another name, assignable through a changeset.
|
||||
ASSIGNABLE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"model_group_alias": "_core_model_group_alias"})
|
||||
|
||||
_MISSING: Final = object()
|
||||
|
||||
|
||||
|
|
@ -1168,16 +1176,16 @@ class RouterHealth:
|
|||
@staticmethod
|
||||
def is_blocked(deployment: "Deployment") -> bool:
|
||||
"""Whether an admin paused this deployment (was ``is_deployment_blocked``)."""
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
return Router._core_is_deployment_blocked(deployment)
|
||||
return PythonCore._core_is_deployment_blocked(deployment)
|
||||
|
||||
@staticmethod
|
||||
def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
|
||||
"""Whether every deployment in the list is paused (was ``are_all_deployments_blocked``)."""
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
return Router._core_are_all_deployments_blocked(deployments)
|
||||
return PythonCore._core_are_all_deployments_blocked(deployments)
|
||||
|
||||
def fully_blocked_model_names(self) -> set[str]: # mutable-ok: the core's type, a fresh set
|
||||
"""Groups whose every deployment is paused (was ``get_fully_blocked_model_names``)."""
|
||||
|
|
@ -1296,13 +1304,17 @@ class RoutingObservers:
|
|||
def apply_changes(router: "Router", changes: ChangeSet) -> ApplyResult:
|
||||
if changes.refresh_model_info:
|
||||
raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()")
|
||||
unknown: Final = sorted(set(changes.assign) - set(SETTING_FIELDS))
|
||||
unknown: Final = sorted(set(changes.assign) - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES})
|
||||
if unknown:
|
||||
raise ValueError(f"not assignable router settings: {unknown}")
|
||||
for name, value in changes.assign.items():
|
||||
setattr(router, "_core_" + name, value)
|
||||
setattr(
|
||||
router,
|
||||
name if name in PLAIN_SETTING_ATTRIBUTES else ASSIGNABLE_ALIASES.get(name, "_core_" + name),
|
||||
detached(value),
|
||||
)
|
||||
if changes.settings:
|
||||
router._core_update_settings(**changes.settings)
|
||||
router._core_update_settings(**{name: detached(value) for name, value in changes.settings.items()})
|
||||
upserted: Final = [router._core_upsert_deployment(deployment=d) for d in changes.upsert]
|
||||
deleted: Final = [router._core_delete_deployment(id=i) for i in changes.delete]
|
||||
if changes.redis_cache is not None:
|
||||
|
|
@ -1388,7 +1400,7 @@ class RoutingContext:
|
|||
return self._router.observe()
|
||||
|
||||
def belongs_to(self, router: "Router") -> bool:
|
||||
return self._router is router
|
||||
return self._router is router or router.__dict__.get("_core") is self._router
|
||||
|
||||
@property
|
||||
def explicit_allowed_fails(self) -> int | None:
|
||||
|
|
|
|||
157
litellm/router_config.py
Normal file
157
litellm/router_config.py
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
"""
|
||||
The router's configuration, built outside the router.
|
||||
|
||||
``RouterConfig`` is static: the constructor arguments, frozen for the router's life.
|
||||
``RouterDynamicConfig`` is a snapshot of the process settings the router reads while it serves
|
||||
(``litellm.num_retries``, ``litellm.model_cost``, the fallback lists, ...). A call reads one snapshot
|
||||
at its start. This module is the only router code that reads ``litellm.*`` settings.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RouterConfig:
|
||||
"""The arguments the router was built with, exactly as passed (unset ones are absent)."""
|
||||
|
||||
arguments: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
|
||||
|
||||
def get(self, name: str, default: object = None) -> object:
|
||||
return self.arguments.get(name, default)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RouterDynamicConfig:
|
||||
"""Process settings the router reads per call. Values are the live objects, not copies."""
|
||||
|
||||
version: int
|
||||
num_retries: int | None
|
||||
max_fallbacks: int | None
|
||||
router_max_fallbacks: int
|
||||
allowed_fails: int
|
||||
request_timeout: float
|
||||
fallbacks: Sequence[object] | None
|
||||
default_fallbacks: Sequence[str] | None
|
||||
context_window_fallbacks: Sequence[object] | None
|
||||
content_policy_fallbacks: Sequence[object] | None
|
||||
drop_params: bool | None
|
||||
provider_list: Sequence[str]
|
||||
expose_router_debug_in_errors: bool
|
||||
redact_user_api_key_info: bool | None
|
||||
model_alias_map: Mapping[str, str]
|
||||
model_cost: Mapping[str, object]
|
||||
known_custom_logger_compatible_callbacks: Sequence[str]
|
||||
exception_types: Sequence[type[BaseException]]
|
||||
callbacks: Sequence[object]
|
||||
success_callback: Sequence[object]
|
||||
failure_callback: Sequence[object]
|
||||
input_callback: Sequence[object]
|
||||
callback_manager: "LoggingCallbackManager"
|
||||
cache: "Cache | None"
|
||||
|
||||
|
||||
_versions: Final = itertools.count(1)
|
||||
|
||||
|
||||
def detached(value: object) -> object:
|
||||
"""``value`` with its plain containers (dict, list, tuple, set) copied, all the way down, so the
|
||||
caller's own objects can't change the router afterwards. Other objects (clients, caches,
|
||||
callbacks, pydantic models) are kept as they are."""
|
||||
if isinstance(value, dict):
|
||||
return {key: detached(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [detached(item) for item in value]
|
||||
if isinstance(value, tuple) and type(value) is tuple:
|
||||
return tuple(detached(item) for item in value)
|
||||
if isinstance(value, set):
|
||||
return {detached(item) for item in value}
|
||||
return value
|
||||
|
||||
|
||||
def build_router_config(arguments: Mapping[str, object]) -> RouterConfig:
|
||||
"""The constructor arguments, detached from the caller's objects: settings are static."""
|
||||
return RouterConfig(arguments=MappingProxyType({name: detached(value) for name, value in arguments.items()}))
|
||||
|
||||
|
||||
def current_dynamic_config() -> RouterDynamicConfig:
|
||||
"""A snapshot of the process settings as they are right now."""
|
||||
return RouterDynamicConfig(
|
||||
version=next(_versions),
|
||||
num_retries=litellm.num_retries,
|
||||
max_fallbacks=litellm.max_fallbacks,
|
||||
router_max_fallbacks=litellm.ROUTER_MAX_FALLBACKS,
|
||||
allowed_fails=litellm.allowed_fails,
|
||||
request_timeout=litellm.request_timeout,
|
||||
fallbacks=litellm.fallbacks,
|
||||
default_fallbacks=litellm.default_fallbacks,
|
||||
context_window_fallbacks=litellm.context_window_fallbacks,
|
||||
content_policy_fallbacks=litellm.content_policy_fallbacks,
|
||||
drop_params=litellm.drop_params,
|
||||
provider_list=litellm.provider_list,
|
||||
expose_router_debug_in_errors=litellm.expose_router_debug_in_errors,
|
||||
redact_user_api_key_info=litellm.redact_user_api_key_info,
|
||||
model_alias_map=litellm.model_alias_map,
|
||||
model_cost=litellm.model_cost,
|
||||
known_custom_logger_compatible_callbacks=litellm._known_custom_logger_compatible_callbacks,
|
||||
exception_types=litellm.LITELLM_EXCEPTION_TYPES,
|
||||
callbacks=litellm.callbacks,
|
||||
success_callback=litellm.success_callback,
|
||||
failure_callback=litellm.failure_callback,
|
||||
input_callback=litellm.input_callback,
|
||||
callback_manager=litellm.logging_callback_manager,
|
||||
cache=litellm.cache,
|
||||
)
|
||||
|
||||
|
||||
_LIVE_SOURCES: Final = MappingProxyType(
|
||||
{
|
||||
"num_retries": "num_retries",
|
||||
"max_fallbacks": "max_fallbacks",
|
||||
"router_max_fallbacks": "ROUTER_MAX_FALLBACKS",
|
||||
"allowed_fails": "allowed_fails",
|
||||
"request_timeout": "request_timeout",
|
||||
"fallbacks": "fallbacks",
|
||||
"default_fallbacks": "default_fallbacks",
|
||||
"context_window_fallbacks": "context_window_fallbacks",
|
||||
"content_policy_fallbacks": "content_policy_fallbacks",
|
||||
"drop_params": "drop_params",
|
||||
"provider_list": "provider_list",
|
||||
"expose_router_debug_in_errors": "expose_router_debug_in_errors",
|
||||
"redact_user_api_key_info": "redact_user_api_key_info",
|
||||
"model_alias_map": "model_alias_map",
|
||||
"model_cost": "model_cost",
|
||||
"known_custom_logger_compatible_callbacks": "_known_custom_logger_compatible_callbacks",
|
||||
"exception_types": "LITELLM_EXCEPTION_TYPES",
|
||||
"callbacks": "callbacks",
|
||||
"success_callback": "success_callback",
|
||||
"failure_callback": "failure_callback",
|
||||
"input_callback": "input_callback",
|
||||
"callback_manager": "logging_callback_manager",
|
||||
"cache": "cache",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class LiveDynamicConfig:
|
||||
"""``RouterDynamicConfig``'s fields read from the process settings at access time, with no snapshot."""
|
||||
|
||||
__slots__ = ()
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
source: Final = _LIVE_SOURCES.get(name)
|
||||
if source is None:
|
||||
raise AttributeError(name)
|
||||
return getattr(litellm, source)
|
||||
|
||||
|
||||
live_dynamic_config: Final = LiveDynamicConfig()
|
||||
56
litellm/router_core.py
Normal file
56
litellm/router_core.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""
|
||||
``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one.
|
||||
|
||||
``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core
|
||||
(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when
|
||||
the router is built, from its ``RouterConfig``.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||
|
||||
from litellm.router_config import RouterConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router_api import (
|
||||
ApplyResult,
|
||||
ChangeSet,
|
||||
RouterHealth,
|
||||
RouterRegistry,
|
||||
RouterSettings,
|
||||
RoutingObservers,
|
||||
)
|
||||
from litellm.router_python_core import PythonCore
|
||||
from litellm.router_python_host import PythonHost
|
||||
from litellm.types.router import CustomRoutingStrategyBase
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RouterCore(Protocol):
|
||||
async def call(self, op: str, request: Mapping[str, object]) -> object: ...
|
||||
|
||||
def registry(self) -> "RouterRegistry": ...
|
||||
|
||||
def settings(self) -> "RouterSettings": ...
|
||||
|
||||
def health(self) -> "RouterHealth": ...
|
||||
|
||||
async def select(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object] | None = None, # mutable-ok: the selection cores take a dict
|
||||
) -> object: ...
|
||||
|
||||
def apply(self, changes: "ChangeSet") -> "ApplyResult": ...
|
||||
|
||||
def observe(self) -> "RoutingObservers": ...
|
||||
|
||||
def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None: ...
|
||||
|
||||
def retire(self) -> None: ...
|
||||
|
||||
|
||||
def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
return (core_class or PythonCore)(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature
|
||||
|
|
@ -31,7 +31,7 @@ from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload
|
||||
|
||||
from litellm.router_api import SETTING_FIELDS, ChangeSet
|
||||
from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Any # noqa: TID251 # only to repeat two old signatures that had Any, unchanged
|
||||
|
|
@ -65,7 +65,7 @@ _INTERNAL_PREFIX: Final = "litellm.router"
|
|||
|
||||
|
||||
# Router's own attribute hooks: a write or delete through an old name passes through them first.
|
||||
_ATTRIBUTE_HOOKS: Final = frozenset({"__setattr__", "__delattr__"})
|
||||
_ATTRIBUTE_HOOKS: Final = frozenset({"__getattribute__", "__setattr__", "__delattr__"})
|
||||
|
||||
|
||||
def _note(name: str, replacement: str) -> None:
|
||||
|
|
@ -88,9 +88,10 @@ def _note(name: str, replacement: str) -> None:
|
|||
def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
|
||||
"""The Router this mixin is part of (it is mixed into ``litellm.Router`` only)."""
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
if isinstance(obj, Router):
|
||||
return obj
|
||||
if isinstance(obj, (Router, PythonCore)):
|
||||
return obj # pyright: ignore[reportReturnType] # a PythonCore carries every member the mixin reads
|
||||
raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
|
||||
|
||||
|
||||
|
|
@ -134,7 +135,7 @@ class _DeprecatedAttribute(Generic[_T]):
|
|||
|
||||
def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
|
||||
_note(self.name, self.replacement)
|
||||
if self.name in SETTING_FIELDS:
|
||||
if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
|
||||
_as_router(obj).apply(ChangeSet(assign={self.name: value}))
|
||||
else:
|
||||
setattr(obj, "_core_" + self.name, value)
|
||||
|
|
@ -169,6 +170,13 @@ _OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] =
|
|||
)
|
||||
|
||||
|
||||
def _override_key(router: object, name: str) -> tuple[int, str]:
|
||||
"""The running-override key for ``router``: a Router and its core share one, so an old member
|
||||
entered on the Router is seen by the override routing that runs on the core."""
|
||||
core: Final = getattr(router, "__dict__", {}).get("_core")
|
||||
return (id(router if core is None else core), name)
|
||||
|
||||
|
||||
def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]:
|
||||
"""
|
||||
A direct call of a subclass override (``router.<name>()``) marks it running, so its
|
||||
|
|
@ -184,7 +192,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
|
|||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
|
||||
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
|
||||
try:
|
||||
return await async_override(router, *args, **kwargs)
|
||||
finally:
|
||||
|
|
@ -198,7 +206,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
|
|||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
|
||||
token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
|
||||
try:
|
||||
return override(router, *args, **kwargs)
|
||||
finally:
|
||||
|
|
@ -236,7 +244,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
|
|||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
key: Final = (id(router), name)
|
||||
key: Final = _override_key(router, name)
|
||||
running: Final = _OVERRIDES_RUNNING.get()
|
||||
if key in running:
|
||||
parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core)
|
||||
|
|
@ -255,7 +263,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
|
|||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: passes the override's arguments through unchanged
|
||||
) -> object:
|
||||
key: Final = (id(router), name)
|
||||
key: Final = _override_key(router, name)
|
||||
running: Final = _OVERRIDES_RUNNING.get()
|
||||
if key in running:
|
||||
return parent(router)(*args, **kwargs)
|
||||
|
|
@ -502,9 +510,9 @@ class DeprecatedRouterMembers:
|
|||
@staticmethod
|
||||
def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool:
|
||||
_note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)")
|
||||
from litellm.router import Router
|
||||
from litellm.router_python_core import PythonCore
|
||||
|
||||
return Router._core_deployment_usable_by_team(model, team_id)
|
||||
return PythonCore._core_deployment_usable_by_team(model, team_id)
|
||||
|
||||
_deployment_usable_by_team = deployment_usable_by_team
|
||||
|
||||
|
|
|
|||
17669
litellm/router_python_core.py
Normal file
17669
litellm/router_python_core.py
Normal file
File diff suppressed because it is too large
Load diff
135
litellm/router_python_host.py
Normal file
135
litellm/router_python_host.py
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
"""
|
||||
``PythonHost``: what the router core reaches outside itself, served from the litellm package.
|
||||
|
||||
The core never touches ``litellm.*`` directly. Inference calls (``invoke``), backoff sleeps,
|
||||
callback registration and the process-wide cache all go through the host, and the process settings
|
||||
come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the same operations.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Literal,
|
||||
TypeAlias,
|
||||
cast, # noqa: TID251 # untyped litellm globals: the live config view and the callback lists
|
||||
)
|
||||
|
||||
from typing_extensions import overload
|
||||
|
||||
import litellm
|
||||
from litellm.router_config import RouterDynamicConfig, live_dynamic_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject
|
||||
from litellm.types.rerank import RerankResponse
|
||||
from litellm.types.utils import (
|
||||
AdapterCompletionStreamWrapper,
|
||||
EmbeddingResponse,
|
||||
ImageResponse,
|
||||
LiteLLMBatch,
|
||||
ModelInfo,
|
||||
ModelResponse,
|
||||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
)
|
||||
|
||||
CallbackList: TypeAlias = Literal[
|
||||
"success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback", "input_callback"
|
||||
]
|
||||
|
||||
Endpoint: TypeAlias = Callable[..., object]
|
||||
|
||||
|
||||
class PythonHost:
|
||||
__slots__ = ()
|
||||
|
||||
# The endpoints whose results the router core reads, typed as litellm declares them; any other
|
||||
# op gets the untyped Endpoint.
|
||||
@overload
|
||||
def invoke(self, op: Literal["acompletion"]) -> Callable[..., Awaitable["ModelResponse | CustomStreamWrapper"]]: ...
|
||||
@overload
|
||||
def invoke(
|
||||
self, op: Literal["atext_completion"]
|
||||
) -> Callable[..., Awaitable["TextCompletionResponse | TextCompletionStreamWrapper"]]: ...
|
||||
@overload
|
||||
def invoke(
|
||||
self, op: Literal["aadapter_completion"]
|
||||
) -> Callable[..., Awaitable["BaseModel | AdapterCompletionStreamWrapper | None"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["aembedding"]) -> Callable[..., Awaitable["EmbeddingResponse"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["aimage_generation"]) -> Callable[..., Awaitable["ImageResponse"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["atranscription"]) -> Callable[..., Awaitable["TranscriptionResponse"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["aspeech"]) -> Callable[..., Awaitable["HttpxBinaryResponseContent"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["arerank"]) -> Callable[..., Awaitable["RerankResponse"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["acreate_file"]) -> Callable[..., Awaitable["OpenAIFileObject"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["acreate_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["aretrieve_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["acancel_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["alist_batches"]) -> Callable[..., Awaitable[object]]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["get_llm_provider"]) -> Callable[..., "tuple[str, str, str | None, str | None]"]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["get_model_info"]) -> Callable[..., "ModelInfo"]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["get_supported_openai_params"]) -> Callable[..., "Sequence[str] | None"]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["token_counter"]) -> Callable[..., int]: ...
|
||||
@overload
|
||||
def invoke(self, op: Literal["calculate_retry_after"]) -> Callable[..., float]: ...
|
||||
@overload
|
||||
def invoke(self, op: str) -> Endpoint: ...
|
||||
def invoke(self, op: str) -> Endpoint:
|
||||
"""The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time."""
|
||||
endpoint: Final[object] = getattr(litellm, op, None)
|
||||
if endpoint is None:
|
||||
raise AttributeError(f"litellm has no endpoint {op!r}")
|
||||
if not callable(endpoint):
|
||||
raise TypeError(f"litellm.{op} is not an endpoint")
|
||||
return endpoint
|
||||
|
||||
def sleep(self, seconds: float) -> Awaitable[None]:
|
||||
return asyncio.sleep(seconds)
|
||||
|
||||
def dynamic(self) -> RouterDynamicConfig:
|
||||
"""The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot)."""
|
||||
return cast(RouterDynamicConfig, live_dynamic_config) # cast-ok: the live view reads the same fields
|
||||
|
||||
@property
|
||||
def callbacks(self) -> "LoggingCallbackManager":
|
||||
return litellm.logging_callback_manager
|
||||
|
||||
def callback_list(self, name: CallbackList) -> list[object]: # mutable-ok: the process callback list itself
|
||||
value: Final = getattr(litellm, name)
|
||||
return cast(list[object], value) # cast-ok: the process callback list is an untyped litellm global
|
||||
|
||||
def set_callback_list(self, name: CallbackList, value: list[object]) -> None: # mutable-ok: assigned as given
|
||||
setattr(litellm, name, value)
|
||||
|
||||
def global_cache(self) -> "Cache | None":
|
||||
return litellm.cache
|
||||
|
||||
def set_global_cache(self, cache: "Cache | None") -> None:
|
||||
litellm.cache = cache
|
||||
|
||||
def suppress_debug_info(self) -> None:
|
||||
litellm.suppress_debug_info = True
|
||||
|
||||
|
||||
python_host: Final = PythonHost()
|
||||
|
|
@ -1,15 +1,20 @@
|
|||
import asyncio
|
||||
import enum
|
||||
import heapq
|
||||
import logging as _logging
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from typing import Final, TypeAlias
|
||||
from typing import Protocol as _Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm import print_verbose
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_logger as _verbose_logger
|
||||
from litellm.caching.caching import DualCache, RedisCache
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError as _RedisCircuitBreakerOpenError
|
||||
from litellm.caching.redis_cache import log_redis_failure as _log_redis_failure
|
||||
from litellm.constants import DEFAULT_IN_MEMORY_TTL, DEFAULT_POLLING_INTERVAL
|
||||
from litellm.exceptions import Timeout
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -30,6 +35,26 @@ class FlowItem(LiteLLMBaseModel):
|
|||
model_name: str
|
||||
|
||||
|
||||
class _QueueCache(_Protocol):
|
||||
async def async_get_cache(self, key: str, *, local_only: bool = False, raise_on_error: bool = False) -> object: ...
|
||||
|
||||
async def async_set_cache(self, key: str, value: object, *, local_only: bool = False) -> object: ...
|
||||
|
||||
|
||||
async def _read_queue(cache: _QueueCache, redis_cache: _QueueCache | None, key: str) -> object:
|
||||
if redis_cache is None:
|
||||
return await cache.async_get_cache(key=key, local_only=True)
|
||||
from redis.exceptions import RedisError as _RedisError
|
||||
|
||||
try:
|
||||
response: Final = await redis_cache.async_get_cache(key=key, raise_on_error=True)
|
||||
except (_RedisError, _RedisCircuitBreakerOpenError) as error:
|
||||
_log_redis_failure(_verbose_logger, _logging.ERROR, "Scheduler queue using local snapshot", error)
|
||||
return await cache.async_get_cache(key=key, local_only=True)
|
||||
await cache.async_set_cache(key=key, value=[] if response is None else response, local_only=True)
|
||||
return response
|
||||
|
||||
|
||||
class Scheduler:
|
||||
cache: DualCache
|
||||
|
||||
|
|
@ -153,7 +178,7 @@ class Scheduler:
|
|||
"""
|
||||
if self.cache is not None:
|
||||
_cache_key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}"
|
||||
response: Final = await self.cache.async_get_cache(key=_cache_key)
|
||||
response: Final = await _read_queue(self.cache, self.cache.redis_cache, _cache_key)
|
||||
if not isinstance(response, list):
|
||||
return []
|
||||
return _QUEUE_ENTRIES.validate_python(response)
|
||||
|
|
|
|||
|
|
@ -288,6 +288,7 @@ class PromptOptions:
|
|||
@dataclass(frozen=True, slots=True, kw_only=True)
|
||||
class ResponseOptions:
|
||||
merge_reasoning_content_in_choices: bool | None = None
|
||||
thought_signature_in_tool_call_id: bool | None = None
|
||||
enable_json_schema_validation: bool | None = None
|
||||
complete_response: bool | None = None
|
||||
keepalive_seconds: float | None = None
|
||||
|
|
|
|||
|
|
@ -751,7 +751,9 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel):
|
|||
thinking_tokens: int | None = None
|
||||
|
||||
|
||||
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"]
|
||||
AnthropicFinishReason = Literal[
|
||||
"end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "pause_turn", "compaction"
|
||||
]
|
||||
|
||||
|
||||
class AnthropicResponse(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class AnthropicMessagesResponse(TypedDict, total=False):
|
|||
model: str | None # This represents the Model type from Anthropic
|
||||
role: Literal["assistant"] | None
|
||||
stop_reason: ReadOnly[
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction", "pause_turn"] | None
|
||||
]
|
||||
stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]]
|
||||
stop_sequence: str | None
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ from pydantic import (
|
|||
FieldSerializationInfo,
|
||||
NonNegativeInt,
|
||||
PrivateAttr,
|
||||
SerializeAsAny,
|
||||
SerializerFunctionWrapHandler,
|
||||
field_serializer,
|
||||
field_validator,
|
||||
|
|
@ -1855,7 +1856,7 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
|
|||
item_id: str
|
||||
output_index: int
|
||||
content_index: int
|
||||
part: BaseLiteLLMOpenAIResponseObject
|
||||
part: SerializeAsAny[BaseLiteLLMOpenAIResponseObject]
|
||||
|
||||
|
||||
class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from typing import Final, Literal, Optional, Union
|
|||
|
||||
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
|
||||
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
|
||||
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
|
||||
from pydantic import ConfigDict, PrivateAttr
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
|
@ -154,7 +155,7 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject):
|
|||
id: str
|
||||
status: str # "completed", "in_progress", etc.
|
||||
role: str # "assistant", "user", etc.
|
||||
content: list[OutputText]
|
||||
content: list[OutputText | ResponseOutputRefusal]
|
||||
phase: Phase = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,20 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|||
|
||||
import httpx
|
||||
from openai import Timeout as SDKTimeout
|
||||
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
BeforeValidator,
|
||||
ConfigDict,
|
||||
Field,
|
||||
JsonValue,
|
||||
StrictInt,
|
||||
ValidationError,
|
||||
ValidationInfo,
|
||||
ValidatorFunctionWrapHandler,
|
||||
WrapValidator,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -531,6 +544,27 @@ def holds_secret_pointer(param_name: str) -> bool:
|
|||
|
||||
_RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"})
|
||||
|
||||
_STRICT_LITELLM_PARAMS: Final = MappingProxyType({"strict_litellm_params": True})
|
||||
|
||||
_StrictlyValidatedParams: Final = TypeVar("_StrictlyValidatedParams", bound=BaseModel)
|
||||
|
||||
|
||||
def validate_litellm_params_strictly(
|
||||
params_model: type[_StrictlyValidatedParams], params: Mapping[str, object]
|
||||
) -> _StrictlyValidatedParams:
|
||||
return params_model.model_validate(params, context=_STRICT_LITELLM_PARAMS)
|
||||
|
||||
|
||||
def _none_when_malformed_outside_strict_validation(
|
||||
value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo
|
||||
) -> object:
|
||||
try:
|
||||
return handler(value)
|
||||
except ValidationError:
|
||||
if info.context is _STRICT_LITELLM_PARAMS:
|
||||
raise
|
||||
return None
|
||||
|
||||
|
||||
_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}")
|
||||
|
||||
|
|
@ -612,6 +646,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
)
|
||||
model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
|
||||
merge_reasoning_content_in_choices: bool | None = False
|
||||
thought_signature_in_tool_call_id: Annotated[
|
||||
bool | None, WrapValidator(_none_when_malformed_outside_strict_validation)
|
||||
] = None
|
||||
model_info: dict | None = None
|
||||
mock_response: str | ModelResponse | Exception | object | None = None
|
||||
|
||||
|
|
|
|||
|
|
@ -677,13 +677,13 @@ class CallTypes(str, Enum):
|
|||
retrieve_fine_tuning_job = "retrieve_fine_tuning_job"
|
||||
responses = "responses"
|
||||
aresponses = "aresponses"
|
||||
alist_input_items = "alist_input_items"
|
||||
get_responses = "get_responses"
|
||||
aget_responses = "aget_responses"
|
||||
delete_responses = "delete_responses"
|
||||
adelete_responses = "adelete_responses"
|
||||
cancel_responses = "cancel_responses"
|
||||
acancel_responses = "acancel_responses"
|
||||
alist_input_items = "alist_input_items"
|
||||
compact_responses = "compact_responses"
|
||||
acompact_responses = "acompact_responses"
|
||||
llm_passthrough_route = "llm_passthrough_route"
|
||||
|
|
|
|||
|
|
@ -1595,6 +1595,7 @@ def dispatch_success_logging(
|
|||
is_litellm_internal_call: bool,
|
||||
) -> None:
|
||||
if is_litellm_internal_call:
|
||||
logging_obj.capture_usage_before_logging(result, start_time, end_time)
|
||||
return
|
||||
|
||||
_schedule_async_success_logging(
|
||||
|
|
@ -1633,9 +1634,14 @@ def _schedule_async_success_logging(
|
|||
if not logging_obj.claim_async_success_log():
|
||||
return
|
||||
|
||||
if logging_obj.defer_async_logging and not is_completion_with_fallbacks:
|
||||
logging_obj.capture_usage_before_logging(result, start_time, end_time, freeze_only=True)
|
||||
|
||||
def _enqueue_async_logging() -> None:
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
|
||||
if not is_completion_with_fallbacks:
|
||||
logging_obj.capture_usage_before_logging(result, start_time, end_time)
|
||||
if logging_obj.defer_async_logging:
|
||||
logging_obj.handle_sync_success_callbacks_for_async_calls(
|
||||
result=result, start_time=start_time, end_time=end_time
|
||||
|
|
@ -2203,6 +2209,7 @@ def client(original_function):
|
|||
|
||||
# LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated
|
||||
verbose_logger.info("Wrapper: Completed Call, calling success_handler")
|
||||
logging_obj.capture_usage_before_logging(TypeAdapter(object).validate_python(result), start_time, end_time)
|
||||
if not is_internal_call.get():
|
||||
# Copy the current context to propagate it to the background thread
|
||||
# This is essential for OpenTelemetry span context propagation
|
||||
|
|
|
|||
|
|
@ -81372,5 +81372,77 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"gpt-6-astra-law": {
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
"input_cost_per_token": 1e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-05,
|
||||
"reasoning_contexts": [
|
||||
"auto",
|
||||
"current_turn",
|
||||
"all_turns"
|
||||
],
|
||||
"reasoning_modes": [
|
||||
"standard",
|
||||
"pro"
|
||||
],
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/batch",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_native_streaming": true,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_cache_breakpoint": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_tool_search": true,
|
||||
"supports_vision": true,
|
||||
"supports_web_search": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"chatgpt/gpt-6-astra-law": {
|
||||
"litellm_provider": "chatgpt",
|
||||
"max_input_tokens": 922000,
|
||||
"max_output_tokens": 128000,
|
||||
"mode": "responses",
|
||||
"source": "https://developers.openai.com/api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
"/v1/responses"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue