scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

Merge remote-tracking branch 'origin/main' into litellm_warn_remaining_reasoning_drops

Browse source
This commit is contained in:
mateo-berri 2026-10-29 16:00:00 +00:00
commit 6043cb9c4f
202 changed files with 13891 additions and 1187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.github/scripts/smoke_test_native_wheel.py vendored
View file

@ -7,6 +7,8 @@ import zipfile
from pathlib import Path
from typing import Final
COST_MAP: Final = Path(__file__).resolve().parents[2] / "model_prices_and_context_window.json"
CHILD_SCRIPT: Final = """
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
@ -34,6 +36,9 @@ else:
after = module.gil_stats()
if not isinstance(after.get("releases"), int):
raise AssertionError(f"native module unusable after panic: {after!r}")
if module.bundled_model_cost_map() != Path(sys.argv[2]).read_bytes():
raise AssertionError("the wheel embeds a different model_prices_and_context_window.json than this checkout")
"""
@ -55,7 +60,9 @@ def main() -> int:
native_path: Final = Path(temporary_directory) / Path(native_members[0].filename).name
native_path.write_bytes(archive.read(native_members[0]))
result: Final = subprocess.run((sys.executable, "-c", CHILD_SCRIPT, str(native_path)), check=False)
result: Final = subprocess.run(
(sys.executable, "-c", CHILD_SCRIPT, str(native_path), str(COST_MAP)), check=False
)
if result.returncode != 0:
sys.stderr.write(f"native wheel smoke test exited with status {result.returncode}\n")

21
.github/workflows/cost-map-guard.yml vendored
View file

@ -46,3 +46,24 @@ jobs:
HEAD_REF: ${{ github.event.pull_request.head.ref }}
run: |
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
- name: Check whether the pull request changes the cost map
id: changed
env:
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if git diff --quiet "$MERGE_BASE" "$HEAD_SHA" -- model_prices_and_context_window.json; then
echo "cost_map=false" >> "$GITHUB_OUTPUT"
else
echo "cost_map=true" >> "$GITHUB_OUTPUT"
fi
- if: steps.changed.outputs.cost_map == 'true'
run: rustup toolchain install --no-self-update
- name: Load the cost map through model-catalog
if: steps.changed.outputs.cost_map == 'true'
env:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
git show "${HEAD_SHA}:model_prices_and_context_window.json" > "$RUNNER_TEMP/model_prices_and_context_window.json"
cargo run --locked --quiet --manifest-path litellm-rust/Cargo.toml -p litellm-model-catalog --bin check-registry \
-- "$RUNNER_TEMP/model_prices_and_context_window.json"

1
.github/workflows/test-unit.yml vendored
View file

@ -322,6 +322,7 @@ jobs:
tests/unit/llms/openai_like
tests/unit/llms/opencode
tests/unit/llms/opensandbox
tests/unit/llms/opensearch
tests/unit/llms/openrouter
tests/unit/llms/ovhcloud
tests/unit/llms/parallel_ai

1
backend/Dockerfile
View file

@ -24,6 +24,7 @@ ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \

11
codecov.yaml
View file

@ -15,9 +15,10 @@ ignore:
flag_management:
default_rules:
carryforward: true
# Dead flags no CI job uploads anymore: their carried-forward sessions were
# measured against old revisions, and the stale line maps mark comment lines
# of since-edited files as missed, sinking patch coverage on unrelated PRs.
# Dead flags no CI job uploads anymore, and flags whose job is path-gated on
# PRs: their carried-forward sessions were measured against old revisions, and
# the stale line maps mark comment lines of since-edited files as missed,
# sinking patch coverage on unrelated PRs.
individual_flags:
- name: proxy-mgmt-behavior
carryforward: false
@ -99,6 +100,10 @@ flag_management:
carryforward: false
- name: proxy-db-endpoints-and-responses
carryforward: false
- name: roi-postgres
carryforward: false
- name: redis-compat
carryforward: false
component_management:
individual_components:

1
gateway/Dockerfile
View file

@ -40,6 +40,7 @@ ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
COPY model_prices_and_context_window.json ./
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \

6
helm/litellm-helm/README.md
View file

@ -259,3 +259,9 @@ itself.
`lensWorker.mode` selects `disabled`, `bundled` or `external`. Bundled mode generates separate service and identity-signing credentials. External mode requires `lensWorker.gateway.secretName` and `lensWorker.serviceTokenSecret.name`; their keys default to `gateway-secret` and `service-token`. Provision distinct values matching the external Lens deployment
Follow the [Lens Helm connection guide](https://github.com/BerriAI/lens/blob/main/helm/lens/README.md#connect-a-gateway) for complete values, verification, GitOps credential requirements and existing-data migration. Source-chart installation requires a built Lens image until the first signed Lens release is published
## Shutdown accounting
`GRACEFUL_SHUTDOWN_TIMEOUT` defaults to 30 seconds. Pending success callbacks share the time remaining since SIGTERM or the first `/health/drain` call. Callback flushing reserves the existing 10-second teardown headroom for cancellation, tracked accounting tasks and persistence. Exhausting this budget can discard callbacks, which is logged
Set `terminationGracePeriodSeconds` to at least `GRACEFUL_SHUTDOWN_TIMEOUT + 10`, allowing additional time for preStop hooks and database writes. This is not a hard shutdown bound or a guarantee of persistence. Database retries retain their own limits, and cancellation-resistant callbacks can remain alive. The existing preStop hook can also be followed by Uvicorn's separate connection drain, as described in `values.yaml`

252
litellm-proxy-extras/litellm_proxy_extras/migration_data_checks.py
View file

@ -14,11 +14,9 @@ either, so the migration keeps today's behavior for it.
"""
import os
import re
from collections.abc import Callable, Iterator
from collections.abc import Callable
from dataclasses import dataclass
from functools import reduce
from itertools import accumulate
from pathlib import Path
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
@ -43,7 +41,7 @@ MIGRATION_DATA_CHECK_TIMEOUT_ENV_VAR: Final = "LITELLM_MIGRATION_DATA_CHECK_TIME
DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS: Final = 600.0
REFUSAL: Final = "Existing rows would make a pending schema change fail:"
Kind = Literal["too_long", "duplicates", "nulls", "any_row", "orphans", "check_violations"]
Kind = Literal["too_long"]
@dataclass(frozen=True, slots=True)
@ -54,9 +52,6 @@ class Check:
statement: str
source: str
index: str | None = None
referenced_table: str | None = None
referenced_columns: tuple[str, ...] = ()
expression: str | None = None
@dataclass(frozen=True, slots=True)
@ -100,33 +95,6 @@ class ColumnType:
ColumnTypeLookup = Callable[[str, str], ColumnType | None]
_IDENT: Final = r'"?(\w+)"?'
_QUALIFIED: Final = r'(?:"?\w+"?\s*\.\s*)?"?(\w+)"?'
_CREATE_INDEX: Final = re.compile(
rf"^\s*CREATE\s+(UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?{_IDENT}\s+ON\s+(?:ONLY\s+)?"
rf"{_QUALIFIED}\s*(?:USING\s+(\w+)\s*)?\((.*?)\)",
re.IGNORECASE | re.DOTALL,
)
_ALTER_TABLE: Final = re.compile(
rf"^\s*ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?(?:ONLY\s+)?{_QUALIFIED}\s+(.*)$", re.IGNORECASE | re.DOTALL
)
_SET_NOT_NULL: Final = re.compile(rf"^ALTER\s+(?:COLUMN\s+)?{_IDENT}\s+SET\s+NOT\s+NULL$", re.IGNORECASE)
_ADD_COLUMN: Final = re.compile(
rf"^ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?!(?:CONSTRAINT|FOREIGN|UNIQUE|CHECK|PRIMARY|EXCLUDE)\b){_IDENT}\s+(.*)$",
re.IGNORECASE | re.DOTALL,
)
_CONSTRAINT: Final = rf"^ADD\s+(?:CONSTRAINT\s+{_IDENT}\s+)?"
_ADD_FOREIGN_KEY: Final = re.compile(
rf"{_CONSTRAINT}FOREIGN\s+KEY\s*\(([^)]*)\)\s*REFERENCES\s+{_QUALIFIED}\s*\(([^)]*)\)(.*)$",
re.IGNORECASE | re.DOTALL,
)
_ADD_UNIQUE: Final = re.compile(rf"{_CONSTRAINT}UNIQUE\s*\(([^)]*)\)", re.IGNORECASE | re.DOTALL)
_ADD_CHECK: Final = re.compile(rf"{_CONSTRAINT}CHECK\s*\((.*)\)\s*(NOT\s+VALID)?\s*$", re.IGNORECASE | re.DOTALL)
_PLAIN_COLUMN: Final = re.compile(rf"^{_IDENT}(?:\s+(?:ASC|DESC|NULLS\s+(?:FIRST|LAST)))*$", re.IGNORECASE)
_COMMENT: Final = re.compile(r"--[^\n]*")
_NOT_NULL: Final = re.compile(r"\bNOT\s+NULL\b", re.IGNORECASE)
_HAS_DEFAULT: Final = re.compile(r"\bDEFAULT\b|\bGENERATED\b", re.IGNORECASE)
def migration_data_check_timeout_seconds() -> float:
"""Seconds one count query may run before the preflight gives up on verifying it."""
@ -145,117 +113,6 @@ def migration_data_check_timeout_seconds() -> float:
return DEFAULT_MIGRATION_DATA_CHECK_TIMEOUT_SECONDS
def statements(migration_sql: str) -> tuple[str, ...]:
"""The statements of one migration file, comments stripped, empty ones dropped."""
parts: Final = _COMMENT.sub("", migration_sql).split(";")
return tuple(part.strip() for part in parts if part.strip())
def _split_top_level(text: str) -> tuple[str, ...]:
"""Split on the commas outside parentheses."""
depth: Final = accumulate((1 if char == "(" else -1 if char == ")" else 0) for char in text)
cuts: Final = tuple(
position for position, (char, level) in enumerate(zip(text, depth, strict=True)) if char == "," and level == 0
)
starts: Final = (0, *(cut + 1 for cut in cuts))
stops: Final = (*cuts, len(text))
pieces: Final = tuple(text[start:stop].strip() for start, stop in zip(starts, stops, strict=True))
return tuple(piece for piece in pieces if piece)
def _plain_columns(column_list: str) -> tuple[str, ...] | None:
"""The column names of an index or constraint column list, None when one entry is an
expression the derivation cannot turn into a count."""
matches: Final = tuple(_PLAIN_COLUMN.match(entry) for entry in _split_top_level(column_list))
if any(match is None for match in matches):
return None
return tuple(match.group(1) for match in matches if match is not None)
def _index_checks(match: "re.Match[str]", statement_text: str, source: str) -> Iterator[Check]:
unique, name, table, method, column_list = match.groups()
columns: Final = _plain_columns(column_list)
if columns is None:
return
label: Final = f'CREATE {"UNIQUE " if unique else ""}INDEX "{name}"'
if method is None or method.lower() == "btree":
yield Check("too_long", table, columns, label, source, index=name)
if unique:
yield Check("duplicates", table, columns, label, source, index=name)
def _alter_table_checks(table: str, action: str, source: str) -> Iterator[Check]:
if (set_not_null := _SET_NOT_NULL.match(action)) is not None:
yield Check("nulls", table, (set_not_null.group(1),), f'ALTER TABLE "{table}" {action}', source)
return
if (add_column := _ADD_COLUMN.match(action)) is not None:
column, body = add_column.groups()
if _NOT_NULL.search(body) is not None and _HAS_DEFAULT.search(body) is None:
yield Check(
"any_row", table, (column,), f'ALTER TABLE "{table}" ADD COLUMN "{column}" ... NOT NULL', source
)
return
if (foreign_key := _ADD_FOREIGN_KEY.match(action)) is not None:
constraint, columns, referenced_table, referenced_columns, tail = foreign_key.groups()
own: Final = _plain_columns(columns)
theirs: Final = _plain_columns(referenced_columns)
if own is None or theirs is None or len(own) != len(theirs) or "NOT VALID" in tail.upper():
return
yield Check(
"orphans",
table,
own,
_constraint_label(table, constraint, "FOREIGN KEY"),
source,
referenced_table=referenced_table,
referenced_columns=theirs,
)
return
if (unique := _ADD_UNIQUE.match(action)) is not None:
constraint, columns = unique.groups()
unique_columns: Final = _plain_columns(columns)
if unique_columns is not None:
yield Check("duplicates", table, unique_columns, _constraint_label(table, constraint, "UNIQUE"), source)
return
if (check := _ADD_CHECK.match(action)) is not None:
constraint, expression, not_valid = check.groups()
if not_valid is None:
yield Check(
"check_violations",
table,
(),
_constraint_label(table, constraint, "CHECK"),
source,
expression=expression,
)
def _constraint_label(table: str, constraint: str | None, kind: str) -> str:
named: Final = f'CONSTRAINT "{constraint}" ' if constraint else ""
return f'ALTER TABLE "{table}" ADD {named}{kind}'
def checks_for_statement(statement_text: str, source: str) -> tuple[Check, ...]:
"""The count queries one SQL statement needs before it can be applied, empty for a
statement that cannot fail on existing rows or that the derivation cannot read."""
if (index := _CREATE_INDEX.match(statement_text)) is not None:
return tuple(_index_checks(index, statement_text, source))
if (alter := _ALTER_TABLE.match(statement_text)) is not None:
table, actions = alter.groups()
return tuple(
check for action in _split_top_level(actions) for check in _alter_table_checks(table, action, source)
) # comprehension-ok: flattening the per-action checks of one statement
return ()
def checks_for_migration(name: str, migration_sql: str) -> tuple[Check, ...]:
return tuple(
check
for statement_text in statements(migration_sql)
for check in checks_for_statement(statement_text, f"migration {name}")
) # comprehension-ok: flattening the per-statement checks of one migration
def checks_for_index(table: str, name: str, columns: tuple[str, ...]) -> tuple[Check, ...]:
"""The entry-size check a request-log index needs before `CREATE INDEX CONCURRENTLY`."""
return (Check("too_long", table, columns, f'CREATE INDEX "{name}"', "request-log index", index=name),)
@ -424,42 +281,7 @@ def _entry_size_query(check: Check, schema: str, types: tuple[ColumnType, ...],
def _count_query(check: Check, schema: str, types: tuple[ColumnType, ...], limit_bytes: int) -> "sql.Composed":
from psycopg import sql
table: Final = sql.Identifier(schema, check.table)
columns: Final = sql.SQL(", ").join(sql.Identifier(column) for column in check.columns)
match check.kind:
case "too_long":
return _entry_size_query(check, schema, types, limit_bytes)
case "duplicates":
not_null: Final = sql.SQL(" AND ").join(
sql.SQL("{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
)
return sql.SQL(
"SELECT count(*) FROM (SELECT 1 FROM {t} WHERE {nn} GROUP BY {cols} HAVING count(*) > 1) AS d"
).format(t=table, nn=not_null, cols=columns)
case "nulls":
return sql.SQL("SELECT count(*) FROM {t} WHERE {col} IS NULL").format(
t=table, col=sql.Identifier(check.columns[0])
)
case "any_row":
return sql.SQL("SELECT count(*) FROM {t}").format(t=table)
case "orphans":
pairs: Final = sql.SQL(" AND ").join(
sql.SQL("parent.{} = child.{}").format(sql.Identifier(theirs), sql.Identifier(own))
for own, theirs in zip(check.columns, check.referenced_columns, strict=True)
)
present: Final = sql.SQL(" AND ").join(
sql.SQL("child.{} IS NOT NULL").format(sql.Identifier(column)) for column in check.columns
)
return sql.SQL(
"SELECT count(*) FROM {t} AS child WHERE {present} AND NOT EXISTS (SELECT 1 FROM {ref} AS parent WHERE {pairs})"
).format(t=table, present=present, ref=sql.Identifier(schema, check.referenced_table or ""), pairs=pairs)
case "check_violations":
return sql.SQL("SELECT count(*) FROM {t} WHERE NOT ({expr})").format(
t=table,
expr=sql.SQL(check.expression or "true"), # pyright: ignore[reportArgumentType] # copied from our own migration file
)
return _entry_size_query(check, schema, types, limit_bytes)
def _applies(
@ -474,15 +296,7 @@ def _applies(
return False
if check.index is not None and _index_is_valid(connection, schema, check.index):
return False
if check.kind == "any_row":
return types is None
if types is None:
return False
if check.kind == "too_long":
return any(column.can_overflow_btree for column in types)
if check.kind == "orphans":
return check.referenced_table is not None and _table_exists(connection, schema, check.referenced_table)
return True
return types is not None and any(column.can_overflow_btree for column in types)
def _too_long_detail(check: Check, types: tuple[ColumnType, ...], values: tuple[object, ...], limit_bytes: int) -> str:
@ -515,21 +329,7 @@ def _run_check(
rows: Final = _int(values[0], 0) if values else 0
if rows == 0:
return None
match check.kind:
case "too_long":
return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
case "duplicates":
return BlockedChange(check, rows, "duplicated value groups")
case "nulls":
return BlockedChange(check, rows, "NULL")
case "any_row":
return BlockedChange(
check, rows, "rows, and a NOT NULL column with no DEFAULT cannot be added to a table with rows"
)
case "orphans":
return BlockedChange(check, rows, f'with no matching row in "{check.referenced_table}"')
case "check_violations":
return BlockedChange(check, rows, f"failing CHECK ({check.expression})")
return BlockedChange(check, rows, _too_long_detail(check, types, values, limit_bytes))
def find_blockers(
@ -571,19 +371,7 @@ def find_blockers(
def _remedy(check: Check) -> str:
joined: Final = ", ".join(check.columns)
match check.kind:
case "too_long":
return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
case "duplicates":
return f'Remove or merge the duplicate "{check.table}" ({joined}) values.'
case "nulls":
return f'Fill the NULL "{check.table}"."{joined}" values, or give the column a default first.'
case "any_row":
return "Give the new column a DEFAULT in the migration, or add it nullable and fill it afterwards."
case "orphans":
return f'Delete or repoint the "{check.table}" ({joined}) values with no "{check.referenced_table}" row.'
case "check_violations":
return f'Fix the "{check.table}" rows that fail the expression before adding the constraint.'
return f'Shorten or hash the long "{check.table}" ({joined}) values so every row\'s index entry fits.'
def _describe(finding: Finding) -> str:
@ -607,30 +395,4 @@ def _describe(finding: Finding) -> str:
def report(findings: tuple[Finding, ...]) -> str:
"""The operator-facing refusal: one block per finding, then what to do next."""
lines: Final = "\n".join(_describe(finding) for finding in findings)
return (
f"{REFUSAL}\n{lines}\n"
"No schema change was made. Fix the rows and start again, or run `litellm --check_migrations_against_existing_data` with the new image to check again."
)
def pending_change_checks(
connection: "psycopg.Connection[tuple[object, ...]]",
schema: str,
migrations_dir: Path | None,
indexes: tuple[tuple[str, str, tuple[str, ...]], ...],
) -> tuple[Check, ...]:
"""Every check the pending migrations and the request-log indexes need. `migrations_dir`
is None when the schema is pushed rather than migrated, so only the indexes are checked."""
migration_checks: Final = (
tuple(
check
for name, migration_sql in pending_migrations(connection, schema, migrations_dir)
for check in checks_for_migration(name, migration_sql)
) # comprehension-ok: flattening the per-migration checks
if migrations_dir is not None
else ()
)
index_checks: Final = tuple(
check for table, name, columns in indexes for check in checks_for_index(table, name, columns)
) # comprehension-ok: flattening the per-index checks
return migration_checks + index_checks
return f"{REFUSAL}\n{lines}\nNo index was built. Fix the rows and the next index build picks it up."

184
litellm-proxy-extras/litellm_proxy_extras/migration_dry_run.py Normal file
View file

@ -0,0 +1,184 @@
"""`litellm --check_migrations_against_existing_data`: run the pending migrations for real, each
inside its own transaction that is always rolled back, so Postgres itself says whether this
database's rows and schema take them. Each migration holds the locks the real upgrade will,
and releases them before the next one runs."""
from __future__ import annotations
import re
import time
from collections.abc import Callable
from dataclasses import dataclass
from functools import reduce
from pathlib import Path
from typing import TYPE_CHECKING, Final
from litellm_proxy_extras.migration_data_checks import pending_migrations
if TYPE_CHECKING:
import psycopg
_CONCURRENTLY: Final = re.compile(
r"\b((?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX|REINDEX\s+\w+)\s+)CONCURRENTLY\s+", re.IGNORECASE
)
UNDEFINED_SQLSTATES: Final = frozenset({"42703", "42P01", "42704", "42883"})
Migration = tuple[str, str]
@dataclass(frozen=True, slots=True)
class Applied:
name: str
seconds: float
replayed: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class Rejected:
name: str
error: str
seconds: float
@dataclass(frozen=True, slots=True)
class Baseline:
"""A database with tables but no `_prisma_migrations`: the boot baselines it instead of replaying
history, so the dry run asks the same question the boot does. `refusal` is None when the boot
will baseline, otherwise its reason (a custom migration folder, or the schema diff)."""
refusal: str | None
@property
def matches(self) -> bool:
return self.refusal is None
@dataclass(frozen=True, slots=True)
class DryRun:
applied: tuple[Applied, ...]
rejected: Rejected | None
baseline: Baseline | None = None
@property
def ok(self) -> bool:
return self.rejected is None and (self.baseline is None or self.baseline.matches)
def transactional(migration_sql: str) -> str:
"""The migration as it can run inside a transaction: `CONCURRENTLY` index builds become plain ones."""
return _CONCURRENTLY.sub(r"\1", migration_sql)
def _attempt(
connection: psycopg.Connection[tuple[object, ...]],
schema: str,
chain: tuple[Migration, ...],
clock: Callable[[], float],
) -> tuple[float, psycopg.Error | None]:
import psycopg
from psycopg import sql
started: Final = clock()
try:
with connection.transaction(force_rollback=True):
connection.execute(sql.SQL("SET LOCAL search_path TO {}").format(sql.Identifier(schema)))
for _, migration_sql in chain:
connection.execute(transactional(migration_sql))
except psycopg.Error as exc:
return clock() - started, exc
return clock() - started, None
def _run(
connection: psycopg.Connection[tuple[object, ...]],
schema: str,
pending: tuple[Migration, ...],
position: int,
done: DryRun,
clock: Callable[[], float],
) -> DryRun:
"""One migration in its own rolled-back transaction. A migration that fails because something
an earlier pending migration creates does not exist yet is rerun with those replayed in front."""
if done.rejected is not None:
return done
name: Final = pending[position][0]
seconds, error = _attempt(connection, schema, (pending[position],), clock)
if error is None:
return DryRun(done.applied + (Applied(name, seconds),), None, done.baseline)
if error.sqlstate not in UNDEFINED_SQLSTATES or position == 0:
return DryRun(done.applied, Rejected(name, str(error).strip(), seconds), done.baseline)
earlier: Final = pending[:position]
chained_seconds, chained_error = _attempt(connection, schema, earlier + (pending[position],), clock)
if chained_error is None:
replayed: Final = tuple(earlier_name for earlier_name, _ in earlier)
return DryRun(done.applied + (Applied(name, chained_seconds, replayed),), None, done.baseline)
return DryRun(done.applied, Rejected(name, str(chained_error).strip(), chained_seconds), done.baseline)
def _has_ledger(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
from psycopg import sql
row: Final = connection.execute(
"SELECT to_regclass(%s)", (sql.Identifier(schema, "_prisma_migrations").as_string(connection),)
).fetchone()
return row is not None and row[0] is not None
def _has_litellm_tables(connection: psycopg.Connection[tuple[object, ...]], schema: str) -> bool:
row: Final = connection.execute(
"SELECT 1 FROM information_schema.tables WHERE table_schema = %s AND table_name LIKE 'LiteLLM\\_%%' LIMIT 1",
(schema,),
).fetchone()
return row is not None
def dry_run(
connection: psycopg.Connection[tuple[object, ...]],
schema: str,
migrations_dir: Path,
baseline_check: Callable[[], str | None] | None = None,
clock: Callable[[], float] = time.monotonic,
) -> DryRun:
"""Apply every pending migration in order, each in its own transaction that is rolled back
before the next one starts. The first migration Postgres rejects ends the run with its error.
`baseline_check` (None when the boot would baseline, otherwise why it refuses) stands in for
the replay on a database that has tables but no migration history, which the boot baselines."""
if baseline_check is not None and not _has_ledger(connection, schema) and _has_litellm_tables(connection, schema):
return DryRun((), None, Baseline(baseline_check()))
pending: Final = pending_migrations(connection, schema, migrations_dir)
return reduce(
lambda done, position: _run(connection, schema, pending, position, done, clock),
range(len(pending)),
DryRun((), None),
)
def _timing(applied: Applied) -> str:
replay: Final = f" (depends on {', '.join(applied.replayed)}, replayed in front of it)" if applied.replayed else ""
return f"\n {applied.name}: {applied.seconds:.1f} s{replay}"
def report(result: DryRun) -> str:
"""What the operator reads: the per-migration time the real upgrade will hold its locks,
or the migration Postgres rejected with Postgres's own words. Nothing was applied either way."""
if result.baseline is not None:
if result.baseline.matches:
return (
"No migration history and the schema matches this build, "
"the proxy baselines it at boot, nothing to dry-run."
)
return f"No migration history and the proxy cannot baseline it at boot: {result.baseline.refusal}"
timings: Final = "".join(_timing(applied) for applied in result.applied)
if result.rejected is None:
count: Final = len(result.applied)
if count == 0:
return "No pending migrations, this database is on this version's schema."
return (
f"{count} pending migration(s) applied one at a time in a dry run, each rolled back before the next "
f"so its locks are released, nothing changed:{timings}"
)
return (
f"Migration {result.rejected.name} fails on this database after {result.rejected.seconds:.1f} s, "
f"nothing was applied.\n{result.rejected.error}"
+ (f"\nApplied before it in the dry run:{timings}" if timings else "")
)

21
litellm-proxy-extras/litellm_proxy_extras/migration_recovery.py
View file

@ -86,6 +86,18 @@ def migration_files(directory: Path) -> tuple[tuple[str, str], ...]:
)
def has_packaged_history(migrations_dir: Path) -> bool:
"""Whether `migrations_dir` carries exactly the migration files and schema.prisma this package ships,
the only history the boot baselines an existing database onto."""
packaged_dir: Final = Path(__file__).parent
migrations: Final = migration_files(migrations_dir)
return (
bool(migrations)
and migrations == migration_files(packaged_dir)
and (migrations_dir / "schema.prisma").read_bytes() == (packaged_dir / "schema.prisma").read_bytes()
)
def baseline_current_schema(
coordinator: MigrationCoordinator,
schema: str,
@ -95,14 +107,9 @@ def baseline_current_schema(
) -> None:
from psycopg import sql
packaged_dir: Final = Path(__file__).parent
migrations: Final = migration_files(migrations_dir)
if (
not migrations
or migrations != migration_files(packaged_dir)
or (migrations_dir / "schema.prisma").read_bytes() != (packaged_dir / "schema.prisma").read_bytes()
):
if not has_packaged_history(migrations_dir):
raise RuntimeError("Cannot automatically baseline an existing database with custom migration history.")
migrations: Final = migration_files(migrations_dir)
coordinator.acquire_prisma_lock()
existing: Final = coordinator.connection.execute(

107
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -11,18 +11,13 @@ from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Final, Optional, Protocol, Union
from typing import TYPE_CHECKING, Final, Protocol
from litellm_proxy_extras import prisma_toolchain
from litellm_proxy_extras._logging import logger
from litellm_proxy_extras._redaction import _redact_credentials
from litellm_proxy_extras.migration_data_checks import (
Finding,
find_blockers,
migration_data_check_timeout_seconds,
pending_change_checks,
report,
)
from litellm_proxy_extras.migration_dry_run import dry_run
from litellm_proxy_extras.migration_dry_run import report as dry_run_report
from litellm_proxy_extras.migration_lock import held_migration_lock
from litellm_proxy_extras.prisma_toolchain import (
PRISMA_COMMAND_TIMEOUT_ENV_VAR,
@ -36,7 +31,6 @@ from litellm_proxy_extras.replica_identity import (
apply_replica_identity_full,
)
from litellm_proxy_extras.request_log_indexes import (
REQUEST_LOG_INDEXES,
ensure_request_log_indexes,
filter_request_log_index_diff,
)
@ -52,7 +46,7 @@ class LensCheckConnect(Protocol):
) -> "psycopg.Connection[tuple[object, ...]]": ...
def str_to_bool(value: Optional[str]) -> bool:
def str_to_bool(value: str | None) -> bool:
if value is None:
return False
return value.lower() in ("true", "1", "t", "y", "yes")
@ -169,7 +163,7 @@ def _without_sql_comments(statement: str) -> str:
).strip()
def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
def _without_spend_logs_pk_clauses(statement: str) -> str | None:
prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
if not prefix_match:
return statement
@ -217,7 +211,7 @@ def _max_migration_timestamp(names) -> int:
return max(_migration_timestamp(n) for n in names)
def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]:
def _redacted_command(command: object) -> str | tuple[str, ...] | list[str]:
if isinstance(command, tuple):
return tuple(_redact_credentials(str(argument)) for argument in command)
if isinstance(command, list):
@ -404,7 +398,7 @@ class ProxyExtrasDBManager:
return tuple(row) if row is not None else ()
@staticmethod
def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]:
def _failed_migration_logs(migration_name: str, started_at: str) -> str | None:
row: Final = ProxyExtrasDBManager._read_migration_ledger(
"SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz "
"AND finished_at IS NULL AND rolled_back_at IS NULL",
@ -721,10 +715,10 @@ class ProxyExtrasDBManager:
return row is not None
@staticmethod
def _prisma_schema_param(url: str) -> Optional[str]:
def _prisma_schema_param(url: str) -> str | None:
"""The `schema` query param Prisma uses to pick its target schema,
or None when the URL does not set one."""
from urllib.parse import urlparse, parse_qsl
from urllib.parse import parse_qsl, urlparse
return next(
(v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
@ -1424,7 +1418,6 @@ class ProxyExtrasDBManager:
bool: True if setup was successful, False otherwise
"""
ensure_prisma_toolchain(prisma_command=_get_prisma_command(), prisma_env=_get_prisma_env())
ProxyExtrasDBManager.raise_if_rows_block_pending_changes(use_migrate=use_migrate)
migrated = ProxyExtrasDBManager._run_migrations(use_migrate=use_migrate, use_v2_resolver=use_v2_resolver)
if not migrated:
return False
@ -1433,46 +1426,69 @@ class ProxyExtrasDBManager:
return True
@staticmethod
def migration_data_check_findings(use_migrate: bool = True) -> tuple[Finding, ...]:
"""What in the existing rows would make a pending migration or request-log index
fail, read with count queries only: no write, no table lock. Empty when nothing
blocks, when no database is configured, or when it cannot be reached, so the
migration step that follows reports the connection problem the way it does today."""
def _baseline_refusal(prisma_dir: Path) -> str | None:
"""Why `baseline_current_schema` would refuse this database at boot, None when it would baseline:
the migrations folder must be the packaged one, then `prisma migrate diff` must find the live
database and this build's schema.prisma identical."""
from litellm_proxy_extras.migration_lock import migration_environment
from litellm_proxy_extras.migration_recovery import has_packaged_history
if not has_packaged_history(prisma_dir):
return (
f"{prisma_dir} is a custom migration history (LITELLM_MIGRATION_DIR), it differs from the "
"migrations this build ships, and the proxy only baselines onto the packaged history."
)
try:
prisma_toolchain.run_prisma(
(
_get_prisma_command(),
"migrate",
"diff",
"--from-schema-datasource",
str(prisma_dir / "schema.prisma"),
"--to-schema-datamodel",
str(prisma_dir / "schema.prisma"),
"--exit-code",
),
timeout=prisma_toolchain.prisma_command_timeout(),
env=migration_environment(_get_prisma_env()),
)
except subprocess.CalledProcessError as exc:
diff: Final = str(exc.stdout or exc.stderr or exc).strip()
return f"the schema differs from this build. prisma migrate diff:\n{diff}"
except subprocess.TimeoutExpired as exc:
return f"prisma migrate diff did not finish within {exc.timeout} s"
return None
@staticmethod
def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]:
"""`--check_migrations_against_existing_data`: the pending migrations applied for real on
DIRECT_URL or DATABASE_URL inside one transaction that is rolled back, so Postgres reports
what the upgrade would hit. (True, the per-migration timings) when they all apply,
(False, Postgres's own error) when one does not."""
import psycopg
if not use_migrate:
return (True, "The schema is pushed rather than migrated, there is nothing to dry-run.")
database_url: Final = os.environ.get("DATABASE_URL")
if not database_url:
return ()
return (False, "DATABASE_URL is not set.")
direct_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(
os.environ.get("DIRECT_URL") or database_url
)
schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
migrations_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir()) / "migrations" if use_migrate else None
indexes: Final = tuple((index.table, index.name, index.columns) for index in REQUEST_LOG_INDEXES)
prisma_dir: Final = Path(ProxyExtrasDBManager._get_prisma_dir())
try:
with psycopg.connect(direct_url, connect_timeout=10, autocommit=True) as connection:
checks: Final = pending_change_checks(connection, schema, migrations_dir, indexes)
return find_blockers(connection, schema, checks, migration_data_check_timeout_seconds())
result: Final = dry_run(
connection,
schema,
prisma_dir / "migrations",
baseline_check=lambda: ProxyExtrasDBManager._baseline_refusal(prisma_dir),
)
except psycopg.OperationalError as exc:
logger.warning(
"Could not reach the database for the pre-migration row check: %s", _redact_credentials(str(exc))
)
return ()
@staticmethod
def migration_data_check_report(use_migrate: bool = True) -> Optional[str]:
"""The operator-facing refusal for `migration_data_check_findings`, None when nothing blocks."""
findings: Final = ProxyExtrasDBManager.migration_data_check_findings(use_migrate=use_migrate)
return report(findings) if findings else None
@staticmethod
def raise_if_rows_block_pending_changes(use_migrate: bool) -> None:
"""Refuse to run migrations that the existing rows would make fail, before any is
applied. The RuntimeError carries the report and reaches the operator through the
proxy's "Database migration cannot proceed" exit."""
refusal: Final = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate)
if refusal is not None:
raise RuntimeError(refusal)
return (False, f"Could not reach the database: {_redact_credentials(str(exc))}")
return (result.ok, dry_run_report(result))
@staticmethod
def build_request_log_indexes(build: Callable[[str, str], bool] = ensure_request_log_indexes) -> bool:
@ -1724,5 +1740,4 @@ class ProxyExtrasDBManager:
time.sleep(random.randrange(5, 15))
finally:
os.chdir(original_dir)
pass
return False

1
litellm-rust/Cargo.lock generated
View file

@ -4345,6 +4345,7 @@ dependencies = [
"litellm-inference-transcription",
"litellm-llms",
"litellm-llms-types",
"litellm-model-catalog",
"litellm-router-types",
"litellm-secrets",
"litellm-secrets-aws",

18
litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs
View file

@ -53,6 +53,7 @@ pub(super) const RULES: &[Rule] = &[
|mapping| {
mapping.error_str.contains("invalid_request_error")
&& !mapping.error_str.contains("Incorrect API key provided")
&& !matches!(mapping.status, Some(401 | 404))
},
PublicError::BadRequest,
),
@ -132,6 +133,23 @@ mod tests {
assert_eq!(classified(Some(400), text), Some(expected));
}
#[rstest::rstest]
#[case::unauthorized(Some(401), None)]
#[case::not_found(Some(404), None)]
#[case::forbidden(Some(403), Some(PublicError::BadRequest))]
#[case::rate_limited(Some(429), Some(PublicError::BadRequest))]
#[case::server_error(Some(500), Some(PublicError::BadRequest))]
#[case::no_status(None, Some(PublicError::BadRequest))]
fn invalid_request_errors_leave_401_and_404_to_the_status_mapping(
#[case] status: Option<u16>,
#[case] expected: Option<PublicError>,
) {
assert_eq!(
classified(status, "invalid_request_error bad field"),
expected
);
}
#[rstest::rstest]
#[case::rate_limit_before_context_window(
"rate limit and This model's maximum context length is 10",

2
litellm-rust/crates/model-catalog/AGENTS.md
View file

@ -25,3 +25,5 @@ A new `N`, image size or resolution in an existing key shape parses with no Rust
## Tests
`BUNDLED_REGISTRY` embeds the root `model_prices_and_context_window.json`, so the tests over it fail a JSON change with a bad value or a key no type reads
`Catalog::problems` is the one definition of a mergeable cost map. The bundled test asserts it is empty, and the cost map guard workflow runs it on a pull request's JSON through `cargo run --bin check-registry <path>`

2
litellm-rust/crates/model-catalog/Cargo.toml
View file

@ -10,6 +10,7 @@ schema = ["dep:schemars", "litellm-cost/schema"]
[dependencies]
litellm-cost = { workspace = true, features = ["serde"] }
litellm-error.workspace = true
indexmap = { version = "2.14.0", features = ["serde"] }
jiff = { version = "0.2.37", default-features = false, features = ["std", "tzdb-bundle-always"] }
@ -24,5 +25,4 @@ time.workspace = true
[dev-dependencies]
jsonschema = { version = "0.55.1", default-features = false }
litellm-error.workspace = true
rstest.workspace = true

26
litellm-rust/crates/model-catalog/src/bin/check-registry.rs Normal file
View file

@ -0,0 +1,26 @@
use std::{env, fs, path::PathBuf, process::ExitCode};
use litellm_model_catalog::Catalog;
fn main() -> ExitCode {
let Some(path) = env::args_os().nth(1).map(PathBuf::from) else {
eprintln!("usage: check-registry <model_prices_and_context_window.json>");
return ExitCode::from(2);
};
let body = match fs::read(&path) {
Ok(body) => body,
Err(error) => {
eprintln!("{}: {error}", path.display());
return ExitCode::FAILURE;
}
};
let problems = Catalog::problems(&body);
for problem in &problems {
eprintln!("{}", litellm_error::Report::new(problem));
}
if problems.is_empty() {
ExitCode::SUCCESS
} else {
ExitCode::FAILURE
}
}

114
litellm-rust/crates/model-catalog/src/catalog.rs
View file

@ -5,7 +5,7 @@ use indexmap::IndexMap;
use serde::Deserialize;
use serde_json::Value;
use crate::error::{Error, Loaded, Rejects, Repr};
use crate::error::{Error, Loaded, Rejects, Repr, pointer_child, pointer_for_key};
use crate::fallback::{FallbackGeneralizations, FallbackRule};
use crate::model::Model;
use crate::store::{Rows, Store};
@ -211,6 +211,36 @@ impl Catalog {
})
}
pub fn problems(body: &[u8]) -> Vec<Error> {
let Loaded {
value: catalog,
errors,
} = match Self::parse(body) {
Ok(loaded) => loaded,
Err(fatal) => return vec![fatal],
};
let unread = catalog.rows().entries.iter().flat_map(|(model, entry)| {
EntryWire::read(entry.fields())
.unknown
.into_iter()
.map(move |(key, _)| {
Error::new(
pointer_child(&pointer_for_key(model), &key),
Repr::UnreadField,
)
})
});
let no_fallback_rules = catalog
.fallback_rules()
.is_none_or(<[_]>::is_empty)
.then(|| Error::from_key(FALLBACK_GENERALIZATIONS, Repr::NoFallbackRules));
errors
.into_iter()
.chain(unread)
.chain(no_fallback_rules)
.collect()
}
pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
let (_, _, errors) = read_row(fields);
errors
@ -404,3 +434,85 @@ fn unknown_timezone(wire: &EntryWire) -> Option<Error> {
)
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
const RULES: &str = r#""fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]}"#;
#[rstest]
#[case::clean(format!(r#"{{{RULES},"m":{{"litellm_provider":"test"}}}}"#), vec![])]
#[case::not_json("{".to_owned(), vec![("", "json")])]
#[case::row_error(format!(r#"{{{RULES},"m":{{}}}}"#), vec![("/m/litellm_provider", "missing_provider")])]
#[case::unread_key(
format!(r#"{{{RULES},"m":{{"litellm_provider":"test","typo_cost":1}}}}"#),
vec![("/m/typo_cost", "unread_field")]
)]
#[case::empty_model_key(
format!(r#"{{{RULES},"":{{"litellm_provider":"test","future_field":true}}}}"#),
vec![("//future_field", "unread_field")]
)]
#[case::unread_key_on_invalid_row(
format!(r#"{{{RULES},"m":{{"typo_cost":1}}}}"#),
vec![("/m/litellm_provider", "missing_provider"), ("/m/typo_cost", "unread_field")]
)]
#[case::escaped_unread_key_on_invalid_row(
format!(r#"{{{RULES},"open/router~v1":{{"typo/cost~":1}}}}"#),
vec![("/open~1router~0v1/litellm_provider", "missing_provider"), ("/open~1router~0v1/typo~1cost~0", "unread_field")]
)]
#[case::no_rules(
r#"{"m":{"litellm_provider":"test"}}"#.to_owned(),
vec![("/fallback_generalizations", "no_fallback_rules")]
)]
#[case::empty_rules(
r#"{"fallback_generalizations":{"rules":[]},"m":{"litellm_provider":"test"}}"#.to_owned(),
vec![("/fallback_generalizations", "no_fallback_rules")]
)]
fn problems_list_what_a_cost_map_change_must_fix(
#[case] body: String,
#[case] expected: Vec<(&str, &str)>,
) {
let problems: Vec<_> = Catalog::problems(body.as_bytes())
.iter()
.map(|problem| (problem.path().to_owned(), <&str>::from(problem.kind())))
.collect();
let expected: Vec<_> = expected
.into_iter()
.map(|(path, kind)| (path.to_owned(), kind))
.collect();
assert_eq!(problems, expected);
}
#[rstest]
fn runtime_parsing_preserves_custom_fields_and_recovers_valid_rows() {
let body = br#"{
"good":{"litellm_provider":"test","custom_metadata":true},
"bad":{"litellm_provider":"test","input_cost_per_token":"wrong"}
}"#;
let loaded = Catalog::parse(body).unwrap();
assert_eq!(loaded.errors.len(), 1);
assert_eq!(loaded.errors[0].path(), "/bad/input_cost_per_token");
let (catalog, rejected) = loaded.value.without_rejected(&loaded.errors);
assert_eq!(rejected.len(), 1);
assert!(catalog.lookup("bad").is_none());
let good = catalog.lookup("good").unwrap();
assert!(good.entry.model().is_some());
assert_eq!(
good.entry.field("custom_metadata"),
Some(&Value::Bool(true))
);
assert!(
Catalog::problems(body)
.iter()
.any(|problem| problem.kind() == crate::ErrorKind::UnreadField)
);
assert!(
Catalog::problems(body)
.iter()
.any(|problem| problem.kind() == crate::ErrorKind::NoFallbackRules)
);
}
}

10
litellm-rust/crates/model-catalog/src/error.rs
View file

@ -109,6 +109,12 @@ pub(crate) enum Repr {
#[source]
cause: Option<DateCause>,
},
#[error("not a field litellm reads")]
#[strum_discriminants(strum(serialize = "unread_field"))]
UnreadField,
#[error("has no rules")]
#[strum_discriminants(strum(serialize = "no_fallback_rules"))]
NoFallbackRules,
}
/// What a problem takes out of the served map
@ -148,7 +154,7 @@ impl Error {
| Repr::NoOffPeakHours
| Repr::EmptyList
| Repr::InvalidDate { .. } => Rejects::Row,
Repr::MissingProvider => Rejects::Nothing,
Repr::MissingProvider | Repr::UnreadField | Repr::NoFallbackRules => Rejects::Nothing,
}
}
}
@ -334,6 +340,8 @@ mod tests {
#[case::no_off_peak_hours(Repr::NoOffPeakHours, Rejects::Row)]
#[case::empty_list(Repr::EmptyList, Rejects::Row)]
#[case::invalid_date(Repr::InvalidDate { cause: None }, Rejects::Row)]
#[case::unread_field(Repr::UnreadField, Rejects::Nothing)]
#[case::no_fallback_rules(Repr::NoFallbackRules, Rejects::Nothing)]
fn policy_classifies_each_error(#[case] error: Repr, #[case] expected: Rejects) {
assert_eq!(Error::from(error).rejects(), expected);
}

18
litellm-rust/crates/model-catalog/tests/catalog.rs
View file

@ -366,18 +366,14 @@ fn written_is_the_document_in_order_including_sections_and_bad_entries() {
}
#[rstest]
fn the_bundled_registry_parses_with_no_problems_and_every_key_understood() {
let loaded = Catalog::parse(BUNDLED_REGISTRY).unwrap();
let catalog = loaded.value;
assert!(loaded.errors.is_empty(), "{:#?}", loaded.errors);
let unrecognized: Vec<_> = catalog.unrecognized().collect();
assert!(unrecognized.is_empty(), "{unrecognized:#?}");
assert!(
catalog
.fallback_rules()
.is_some_and(|rules| !rules.is_empty())
);
fn the_bundled_registry_has_no_problems() {
let problems: Vec<_> = Catalog::problems(BUNDLED_REGISTRY)
.iter()
.map(report)
.collect();
assert!(problems.is_empty(), "{problems:#?}");
const REFERENCE_MODEL_COUNT: usize = 4303;
let catalog = Catalog::parse(BUNDLED_REGISTRY).unwrap().value;
assert!(catalog.model_count() as f64 >= REFERENCE_MODEL_COUNT as f64 * 0.9);
}

12
litellm-rust/crates/python-bridge/AGENTS.md
View file

@ -106,6 +106,18 @@ GIL handling to `litellm-host-python`.
## Data Handling
- Reuse large immutable bundled data and parsed catalogs across calls, including in
replacement APIs. Share Rust storage and cache repeated Python projections instead
of reparsing, serializing or copying the full payload on every access
- Cache owned Python objects with `PyOnceLock<Py<T>>`, then bind and clone references
while attached. Do not use blocking `OnceLock::get_or_init` while attached
([PyO3 initialization](https://pyo3.rs/v0.29.2/faq.html),
[reference ownership](https://pyo3.rs/v0.29.2/types.html))
- Never reuse Python objects across interpreters or interpreter reinitialization.
Once-only caches are for immutable data. Reloadable/custom data needs versioned
entries or invalidation, and caller-mutable Python containers must not be shared
- Test unchanged contents and allocation reuse, not just equal values. Keep this
coverage when replacing a bundled-data accessor with a catalog API
- OCR payloads can contain personal data and large base64 images. Do not log
payloads or provider responses.
- Avoid copying large payloads more than needed. The current JSON round-trip is

1
litellm-rust/crates/python-bridge/Cargo.toml
View file

@ -49,6 +49,7 @@ litellm-secrets-types.workspace = true
litellm-llms-types.workspace = true
litellm-router-types.workspace = true
litellm-host-python.workspace = true
litellm-model-catalog.workspace = true
litellm-token-counter = { path = "../token-counter", default-features = false }
pyo3.workspace = true
prost.workspace = true

4
litellm-rust/crates/python-bridge/src/lib.rs
View file

@ -8,6 +8,7 @@ mod execution;
mod http;
mod logger;
mod marshal;
mod model_registry;
mod preflight;
mod python_settings;
mod routes;
@ -30,6 +31,8 @@ mod _native {
#[pymodule_export]
use crate::logger::NativeDiagnosticProcessor;
#[pymodule_export]
use crate::model_registry::bundled_model_cost_map;
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
use crate::routes::chat_completions::{acompletion, completion};
@ -110,6 +113,7 @@ mod tests {
"NativeClickHouseSpendConfig",
"NativeClickHouseSpendStorage",
"trace_encode_error",
"bundled_model_cost_map",
"TokenCounter",
"Tokenizer",
"gil_stats",

30
litellm-rust/crates/python-bridge/src/model_registry.rs Normal file
View file

@ -0,0 +1,30 @@
use pyo3::prelude::*;
use pyo3::sync::PyOnceLock;
use pyo3::types::PyBytes;
#[pyfunction]
pub fn bundled_model_cost_map(py: Python<'_>) -> Bound<'_, PyBytes> {
static BYTES: PyOnceLock<Py<PyBytes>> = PyOnceLock::new();
BYTES
.get_or_init(py, || {
PyBytes::new(py, litellm_model_catalog::BUNDLED_REGISTRY).unbind()
})
.bind(py)
.clone()
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
fn repeated_calls_reuse_the_bundled_registry_bytes() {
Python::initialize();
let first = Python::attach(|py| bundled_model_cost_map(py).unbind());
Python::attach(|py| {
let second = bundled_model_cost_map(py);
assert!(first.bind(py).is(&second));
assert_eq!(second.as_bytes(), litellm_model_catalog::BUNDLED_REGISTRY);
});
}
}

17
litellm/exceptions.py
View file

@ -139,6 +139,7 @@ class AuthenticationError(openai.AuthenticationError):
litellm_debug_info: str | None = None,
max_retries: int | None = None,
num_retries: int | None = None,
body: Mapping[str, object] | None = None,
):
self.status_code = 401
self.message = f"litellm.AuthenticationError: {message}"
@ -152,7 +153,7 @@ class AuthenticationError(openai.AuthenticationError):
request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object
)
super().__init__(
self.message, response=self.response, body=None
self.message, response=self.response, body=body
) # Call the base class constructor with the parameters it needs
def __str__(self):
@ -189,6 +190,7 @@ class NotFoundError(openai.NotFoundError):
litellm_debug_info: str | None = None,
max_retries: int | None = None,
num_retries: int | None = None,
body: Mapping[str, object] | None = None,
):
self.status_code = 404
self.message = f"litellm.NotFoundError: {message}"
@ -202,7 +204,7 @@ class NotFoundError(openai.NotFoundError):
request=httpx.Request(method="GET", url="https://litellm.ai"), # mock request object
)
super().__init__(
self.message, response=self.response, body=None
self.message, response=self.response, body=body
) # Call the base class constructor with the parameters it needs
def __str__(self):
@ -321,6 +323,17 @@ class UnsupportedEndpointError(BadRequestError):
pass
class EndpointNotSupportedByModelError(BadRequestError):
def __init__(self, message: str, model: str, llm_provider: str = "") -> None:
super().__init__( # pyright: ignore[reportUnknownMemberType] # inherited public exception constructor is untyped
message=message, model=model, llm_provider=llm_provider
)
self.message = message
self.type = "invalid_request_error"
self.param = "model"
self.code = "400"
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,

12
litellm/images/main.py
View file

@ -68,6 +68,11 @@ from litellm.utils import (
)
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
_PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE: Final = (
LlmProviders.DASHSCOPE,
LlmProviders.QWENCLOUD,
LlmProviders.QWEN_AI_PLATFORM,
)
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
@ -271,6 +276,7 @@ def image_generation(
"""
try:
args: Final = locals()
requested_api_base: Final = api_base
aimg_generation: Final = kwargs.get("aimg_generation", False)
litellm_call_id: Final = kwargs.get("litellm_call_id", None)
logger_fn: Final = kwargs.get("logger_fn", None)
@ -468,7 +474,11 @@ def image_generation(
raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
# Resolve api_base from litellm.api_base if not explicitly provided
_api_base: Final = api_base or litellm.api_base
_api_base: Final = (
requested_api_base
if custom_llm_provider in _PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE
else api_base or litellm.api_base
)
litellm_params_dict["api_base"] = _api_base
return llm_http_handler.image_generation_handler(

19
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -413,6 +413,25 @@ def _map_openai_exception(
body=getattr(original_exception, "body", None),
)
elif "invalid_request_error" in error_str and "Incorrect API key provided" not in error_str:
invalid_request_status: Final = getattr(original_exception, "status_code", None)
if invalid_request_status == 401:
raise AuthenticationError(
message=f"{exception_provider} - {message}",
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
body=getattr(original_exception, "body", None),
)
if invalid_request_status == 404:
raise NotFoundError(
message=f"{exception_provider} - {message}",
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
body=getattr(original_exception, "body", None),
)
raise BadRequestError(
message=f"{exception_provider} - {message}",
llm_provider=custom_llm_provider,

8
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -31,7 +31,13 @@ EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = Mapping
)
def native_health_check_mode(model: str, custom_llm_provider: str | None) -> Literal["anthropic_messages"] | None:
def native_health_check_mode(
model: str, custom_llm_provider: str | None
) -> Literal["anthropic_messages", "realtime"] | None:
if custom_llm_provider == "soniox":
from litellm.llms.soniox.common_utils import soniox_health_check_mode
return soniox_health_check_mode(model)
if custom_llm_provider != "bedrock_mantle":
return None
from litellm.llms.bedrock_mantle.common_utils import mantle_health_check_mode

33
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
from litellm.types.llms.databricks import DatabricksTool
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
ChatCompletionThinkingBlock,
ImageURLListItem,
@ -59,6 +60,31 @@ _CHOICES_FIELDS: Final[frozenset] = frozenset(Choices.model_fields.keys())
_MODEL_RESPONSE_FIELDS: Final[frozenset] = frozenset(ModelResponse.model_fields.keys()) | {"usage"}
_ANNOTATION_ENTRIES: Final = TypeAdapter(tuple[object, ...])
def _is_provider_specific_annotation(annotation: object) -> bool:
match annotation:
case {"type": "url_citation"}:
return False
case {"type": _}:
return True
case _:
return False
def _annotation_entries(annotations: object) -> tuple[object, ...]:
return _ANNOTATION_ENTRIES.validate_python(annotations) if isinstance(annotations, list) else ()
def partition_annotations(annotations: object) -> tuple[object, tuple[object, ...]]:
entries: Final = _annotation_entries(annotations)
others: Final = tuple(annotation for annotation in entries if _is_provider_specific_annotation(annotation))
if not others:
return annotations, ()
return [annotation for annotation in entries if not _is_provider_specific_annotation(annotation)] or None, others
def _normalize_images_for_message(
images: list[dict] | None,
) -> list[ImageURLListItem] | None:
@ -787,6 +813,9 @@ def convert_to_model_response_object(
provider_specific_fields = dict(choice["message"].get("provider_specific_fields", None) or {})
for f in choice["message"].keys() - _MESSAGE_FIELDS:
provider_specific_fields[f] = choice["message"][f]
annotations, other_annotations = partition_annotations(choice["message"].get("annotations", None))
if other_annotations:
provider_specific_fields["annotations"] = list(other_annotations)
# Handle reasoning models that display `reasoning_content` within `content`
reasoning_content, content = extract_reasoning_content(choice["message"])
@ -809,7 +838,9 @@ def convert_to_model_response_object(
reasoning_content=reasoning_content,
thinking_blocks=thinking_blocks,
reasoning_items=choice["message"].get("reasoning_items", None),
annotations=choice["message"].get("annotations", None),
annotations=cast( # cast-ok: provider annotation dicts pass through whole, keeping keys such as url_citation.content
list[ChatCompletionAnnotation] | None, annotations
),
images=_normalize_images_for_message(choice["message"].get("images", None)),
)
finish_reason = choice.get("finish_reason", None)

125
litellm/litellm_core_utils/logging_worker.py
View file

@ -6,7 +6,7 @@ import atexit
import contextvars
import inspect
import logging
from collections.abc import Coroutine, Iterator
from collections.abc import Awaitable, Callable, Coroutine, Iterator
from typing import Final
from typing_extensions import TypedDict
@ -66,14 +66,19 @@ class LoggingWorker:
max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE,
concurrency: int = LOGGING_WORKER_CONCURRENCY,
timeout_summary_window: float = LOGGING_WORKER_TIMEOUT_SUMMARY_WINDOW_SECONDS,
*,
retry_sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
):
self.timeout = timeout
self.max_queue_size = max_queue_size
self.concurrency = concurrency
self.timeout_summary_window = timeout_summary_window
self._queue: asyncio.Queue[LoggingTask] | None = None
self._worker_task: asyncio.Task | None = None
self._running_tasks: set[asyncio.Task] = set()
self._worker_task: asyncio.Task[None] | None = None
self._running_tasks: set[asyncio.Task[None]] = set()
self._overflow_tasks: set[asyncio.Task[None]] = set() # mutable-ok: done callbacks discard refs
self._pending_enqueues: dict[int, LoggingTask] = {} # mutable-ok: retries and shutdown share callback ownership
self._retry_sleep = retry_sleep
self._dequeued_tasks: dict[int, LoggingTask] = {} # mutable-ok: refs so flush can rescue never-started tasks
self._sem: asyncio.Semaphore | None = None
self._bound_loop: asyncio.AbstractEventLoop | None = None
@ -168,6 +173,7 @@ class LoggingWorker:
self._sem = None
self._worker_task = None
self._running_tasks.clear()
self._overflow_tasks.clear()
self._timeout_summary_task = None
self._timeout_burst_count = 0
self._queue = new_queue
@ -185,6 +191,10 @@ class LoggingWorker:
self._sem = asyncio.Semaphore(self.concurrency)
if self._worker_task is None or self._worker_task.done():
self._worker_task = asyncio.create_task(self._worker_loop())
if not self._overflow_tasks:
self._aggressive_clear_in_progress = False
for pending in tuple(self._pending_enqueues.values()):
self._schedule_delayed_enqueue_retry(pending)
async def _process_log_task(
self, task: LoggingTask, sem: asyncio.Semaphore, queue: "asyncio.Queue[LoggingTask]"
@ -278,12 +288,24 @@ class LoggingWorker:
# Capture the current context when enqueueing
task: Final = LoggingTask(coroutine=coroutine, context=contextvars.copy_context())
self._enqueue_task(task)
def _enqueue_task(self, task: LoggingTask) -> None:
if self._queue is None:
return
try:
self._queue.put_nowait(task)
except asyncio.QueueFull:
# Queue is full - handle it appropriately
verbose_logger.exception("LoggingWorker queue is full")
self._pending_enqueues[id(task)] = task
self._handle_queue_full(task)
else:
self._pending_enqueues.pop(id(task), None)
def _start_overflow_task(self, coroutine: Coroutine[object, object, None]) -> None:
task: Final = asyncio.create_task(coroutine)
self._overflow_tasks.add(task)
task.add_done_callback(self._overflow_tasks.discard)
def _should_start_aggressive_clear(self) -> bool:
"""
@ -326,7 +348,7 @@ class LoggingWorker:
if self._should_start_aggressive_clear():
self._mark_aggressive_clear_started()
# Schedule clearing as async task so enqueue returns immediately (non-blocking)
asyncio.create_task(self._aggressively_clear_queue_async(task))
self._start_overflow_task(self._aggressively_clear_queue_async(task))
else:
# Cooldown active or clear in progress, schedule a delayed retry
self._schedule_delayed_enqueue_retry(task)
@ -363,7 +385,7 @@ class LoggingWorker:
delay: Final = self._calculate_retry_delay()
# Schedule the retry as a background task
asyncio.create_task(self._retry_enqueue_task(task, delay))
self._start_overflow_task(self._retry_enqueue_task(task, delay))
except RuntimeError:
# No event loop, drop the task as we can't schedule a retry
pass
@ -373,17 +395,8 @@ class LoggingWorker:
Retry enqueueing the task after delay, preserving original context.
This is called as a background task from _schedule_delayed_enqueue_retry.
"""
await asyncio.sleep(delay)
# Try to enqueue the task directly, preserving its original context
if self._queue is None:
return
try:
self._queue.put_nowait(task)
except asyncio.QueueFull:
# Still full - handle it appropriately (clear or retry again)
self._handle_queue_full(task)
await self._retry_sleep(delay)
self._enqueue_task(task)
def _extract_tasks_from_queue(self) -> list[LoggingTask]:
"""
@ -423,15 +436,10 @@ class LoggingWorker:
return
extracted_tasks: Final = self._extract_tasks_from_queue()
# Add new task to extracted tasks to process directly
if new_task is not None:
self._track_dequeued(new_task)
extracted_tasks.append(new_task)
# Process extracted tasks directly
self._enqueue_task(new_task)
if extracted_tasks:
await self._process_extracted_tasks(extracted_tasks)
await asyncio.gather(*(self._process_single_task(task) for task in extracted_tasks))
except Exception as e:
verbose_logger.exception("LoggingWorker error during aggressive clear: %s", e)
finally:
@ -439,7 +447,7 @@ class LoggingWorker:
self._aggressive_clear_in_progress = False
async def _process_single_task(self, task: LoggingTask) -> None:
"""Process a single task and mark it done."""
"""Process a task extracted from the queue and mark it done."""
if self._queue is None:
return
@ -455,17 +463,6 @@ class LoggingWorker:
self._untrack_dequeued(task)
self._queue.task_done()
async def _process_extracted_tasks(self, tasks: list[LoggingTask]) -> None:
"""
Process tasks that were extracted from the queue to make room.
Processes them concurrently without semaphore limits for maximum speed.
"""
if not tasks or self._queue is None:
return
# Process all tasks concurrently for maximum speed
await asyncio.gather(*[self._process_single_task(task) for task in tasks])
def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, None, None]) -> None:
"""
Ensure the logging worker is initialized and enqueue the coroutine.
@ -473,32 +470,63 @@ class LoggingWorker:
self.start()
self.enqueue(async_coroutine)
async def stop(self) -> None:
@property
def pending_count(self) -> int:
return (
(self._queue.qsize() if self._queue is not None else 0)
+ len(self._dequeued_tasks)
+ len(self._pending_enqueues)
)
async def stop(self, *, discard_pending: bool = False, timeout: float | None = None) -> None:
"""Stop the logging worker and clean up resources."""
if discard_pending and self._queue is not None:
discarded: Final = self._drain_pending(self._queue)
for task in discarded:
task["coroutine"].close()
self._queue.task_done()
if discarded:
verbose_logger.warning("LoggingWorker shutdown: %d queued callbacks discarded", len(discarded))
if self._timeout_summary_task is not None:
self._timeout_summary_task.cancel()
self._timeout_summary_task = None
self._emit_timeout_summary()
if self._worker_task is None and not self._running_tasks:
# No worker launched and no in-flight tasks to drain.
return
if discard_pending:
rejected: Final = tuple(self._pending_enqueues.values())
self._pending_enqueues.clear()
for task in rejected:
task["coroutine"].close()
if rejected:
verbose_logger.warning("LoggingWorker shutdown: %d overflow callbacks discarded", len(rejected))
tasks_to_cancel: Final[list[asyncio.Task]] = list(self._running_tasks)
if self._worker_task:
# Include the main worker loop so it stops fetching work.
tasks_to_cancel.append(self._worker_task)
tasks_to_cancel: Final = (
*self._running_tasks,
*self._overflow_tasks,
*((self._worker_task,) if self._worker_task is not None else ()),
)
if not tasks_to_cancel:
return
for task in tasks_to_cancel:
# Propagate cancellation to every pending task.
task.cancel()
# Wait for cancellation to settle; ignore errors raised during shutdown.
await asyncio.gather(*tasks_to_cancel, return_exceptions=True)
done, pending = await asyncio.wait(tasks_to_cancel, timeout=timeout)
await asyncio.gather(*done, return_exceptions=True)
if pending:
verbose_logger.warning("LoggingWorker shutdown: %d tasks still alive after cancellation", len(pending))
self._worker_task = None
if discard_pending and self._queue is not None:
for task in self._unstarted_dequeued_tasks():
task["coroutine"].close()
self._untrack_dequeued(task)
self._queue.task_done()
if self._worker_task is not None and self._worker_task.done():
self._worker_task = None
# Drop references to completed tasks so we can restart cleanly.
self._running_tasks.clear()
self._running_tasks.difference_update(done)
async def flush(self) -> None:
"""Flush the logging queue.
@ -516,6 +544,9 @@ class LoggingWorker:
return
self.start()
await self._queue.join()
while self._overflow_tasks:
await asyncio.wait(tuple(self._overflow_tasks))
await self._queue.join()
async def clear_queue(self):
"""

103
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -424,6 +424,109 @@ def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) ->
return None
@dataclass(frozen=True, slots=True)
class InlineDocument:
filename: str
file_data: str
def _document_media_type(source: object) -> str:
match source:
case {"media_type": str() as media_type} if media_type:
return media_type
case _:
return "application/pdf"
def anthropic_inline_document(document: Mapping[str, object]) -> InlineDocument | None:
source: Final = document.get("source")
match source:
case {"type": "base64", "data": str() as data} if data:
title: Final = document.get("title")
return InlineDocument(
filename=title if isinstance(title, str) and title else "document.pdf",
file_data=f"data:{_document_media_type(source)};base64,{data}",
)
case _:
return None
def is_pdf_file_part(part: object) -> bool:
match part:
case {"type": "file", "file": {"file_data": str() as file_data}} if file_data.startswith("data:"):
return file_data.startswith("data:application/pdf")
case {"type": "file", "file": {"format": str() as file_format}}:
return file_format == "application/pdf"
case {"type": "file", "file": {"filename": str() as filename}}:
return filename.lower().endswith(".pdf")
case _:
return False
_CONTENT_PARTS: Final = TypeAdapter(tuple[object, ...])
def content_parts(message: Mapping[str, object]) -> Sequence[object]:
content: Final = message.get("content")
return _CONTENT_PARTS.validate_python(content) if isinstance(content, list) else ()
def _has_pdf_file_part(message: Mapping[str, object]) -> bool:
return any(is_pdf_file_part(part) for part in content_parts(message))
def messages_have_pdf_file_part(messages: Sequence[Mapping[str, object]]) -> bool:
return any(_has_pdf_file_part(message) for message in messages)
def _message_without_pdf_file_parts(message: AllMessageValues) -> AllMessageValues:
if not _has_pdf_file_part(message):
return message
kept: Final = [part for part in content_parts(message) if not is_pdf_file_part(part)]
return cast(AllMessageValues, {**message, "content": kept}) # cast-ok: same message with fewer content parts
def _is_pdf_only_user_turn(message: AllMessageValues) -> bool:
parts: Final = content_parts(message)
return message["role"] == "user" and bool(parts) and all(is_pdf_file_part(part) for part in parts)
def messages_without_pdf_file_parts(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]:
return tuple(
_message_without_pdf_file_parts(message) for message in messages if not _is_pdf_only_user_turn(message)
)
_CONTENT_PART: Final = TypeAdapter(Mapping[str, object])
def _file_part_as_image_url(part: object) -> object:
match part:
case {"type": "file", "file": {"file_data": str() as file_data}} if file_data:
fields: Final = _CONTENT_PART.validate_python(part)
return {key: value for key, value in fields.items() if key != "file"} | {
"type": "image_url",
"image_url": {"url": file_data},
}
case _:
return part
def message_with_file_parts_as_image_urls(message: Mapping[str, object]) -> Mapping[str, object]:
parts: Final = content_parts(message)
if not parts:
return message
return {**message, "content": [_file_part_as_image_url(part) for part in parts]}
def _typed_message_with_file_parts_as_image_urls(message: AllMessageValues) -> AllMessageValues:
return cast(AllMessageValues, message_with_file_parts_as_image_urls(message)) # cast-ok: same message, image parts
def messages_with_file_parts_as_image_urls(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]:
return tuple(_typed_message_with_file_parts_as_image_urls(message) for message in messages)
def _image_part_url(part: Mapping[str, object]) -> str | None:
"""The image URL carried by one content part, whichever of the three dialects wrote it."""
part_type: Final = part.get("type")

26
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -20,6 +20,7 @@ from litellm.constants import (
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
from litellm.types.llms.openai import (
OpenAIRealtimeEvents,
@ -1281,24 +1282,29 @@ class RealTimeStreaming:
verbose_logger.exception("Connection closed in backend to client send messages - %s", e)
close: Final = backend_close_from(e)
self._flush_unbilled_transcription_usage()
if self._backend_refused_session(close):
await self.log_backend_refusal(e)
else:
await self.log_messages()
await self._log_session_end(e if self._backend_refused_session(close) else None)
return close
except asyncio.CancelledError:
self._flush_unbilled_transcription_usage()
await self.log_messages()
await self._log_session_end(None)
raise
except Exception as e:
verbose_logger.exception("Error in backend to client send messages: %s", e)
self._flush_unbilled_transcription_usage()
await self.log_messages()
await self._log_session_end(None)
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
return close.code != 1000 and not self._backend_sent_frames
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
refusal: Final = provider_refusal or transport_refusal
if refusal is None:
await self.log_messages()
return
await self.log_backend_refusal(refusal)
async def log_backend_refusal(self, error: Exception) -> None:
if not self.logging_obj:
return
@ -1691,7 +1697,13 @@ class RealTimeStreaming:
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
sent = await self._send_to_backend(message)
try:
sent = await self._send_to_backend(message)
except RealtimeTranscriptionProtocolError as e:
await self.websocket.send_text(realtime_error_event(str(e), error_type="invalid_request_error"))
if self._uses_deferred_backend_setup():
raise
continue
if guardrail_turn_detection_injected and sent:
self._guardrail_turn_detection_update_sent = True

60
litellm/litellm_core_utils/served_output_texts.py
View file

@ -4,12 +4,13 @@ post-call guardrail rewrote rather than the provider response the proxy assemble
from __future__ import annotations
from collections.abc import Sequence
from itertools import chain
from typing import Final, Literal
from pydantic import BaseModel, Field, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ModelResponse, ModelResponseStream
from litellm.types.utils import ModelResponse, ModelResponseStream, TextCompletionResponse
SERVED_OUTPUT_TEXTS_KEY: Final = "served_output_texts"
@ -44,6 +45,19 @@ class _ChatChoices(LiteLLMBaseModel):
choices: list[object]
class _TextCompletionChoice(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
index: int | None = None
text: str | None = None
class _TextCompletionChunk(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
choices: tuple[_TextCompletionChoice, ...]
def _as_json_object(response: object) -> dict[str, object] | None:
candidate: Final = response.model_dump() if isinstance(response, BaseModel) else response
try:
@ -108,8 +122,13 @@ def served_output_texts(response: object) -> ServedTexts | None:
def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None:
if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks):
return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)))
chat_chunks: Final = tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream))
if chunks and len(chat_chunks) == len(chunks):
return _indexed_texts_by_choice(tuple(chain.from_iterable(_chat_chunk_texts(chunk) for chunk in chat_chunks)))
if chunks and all(isinstance(chunk, TextCompletionResponse) for chunk in chunks):
return _indexed_texts_by_choice(
tuple(chain.from_iterable(_text_completion_chunk_texts(chunk) for chunk in chunks))
)
from litellm.llms.anthropic.messages.stream_assembly import (
assemble_anthropic_sse_stream,
is_anthropic_sse_stream,
@ -121,19 +140,26 @@ def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None:
return _chat_texts(assembled) if assembled is not None else None
def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int) -> str | None:
contents: Final = tuple(
content
for chunk in chunks
def _chat_chunk_texts(chunk: ModelResponseStream) -> tuple[tuple[int, str | None], ...]:
return tuple(
(choice.index or 0, content if isinstance(content := choice.delta.content, str) else None)
for choice in chunk.choices
if (choice.index or 0) == index and isinstance(content := choice.delta.content, str)
)
return "".join(contents) if contents else None
def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None:
choice_count: Final = max(((choice.index or 0) + 1 for chunk in chunks for choice in chunk.choices), default=0)
texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count))
def _text_completion_chunk_texts(chunk: object) -> tuple[tuple[int, str | None], ...]:
parsed: Final = _TextCompletionChunk.model_validate(_as_json_object(chunk))
return tuple((choice.index or 0, choice.text) for choice in parsed.choices)
def _joined_choice_text(indexed_texts: Sequence[tuple[int, str | None]], index: int) -> str | None:
texts: Final = tuple(text for text_index, text in indexed_texts if text_index == index and text is not None)
return "".join(texts) if texts else None
def _indexed_texts_by_choice(indexed_texts: Sequence[tuple[int, str | None]]) -> ServedTexts | None:
choice_count: Final = max((index + 1 for index, _ in indexed_texts), default=0)
texts: Final = tuple(_joined_choice_text(indexed_texts, index) for index in range(choice_count))
return texts if any(text is not None for text in texts) else None
@ -170,6 +196,8 @@ def _choice_with_text(choice: object, text: str | None) -> object:
if choice_obj is None or text is None:
return choice
message: Final = _as_json_object(choice_obj.get("message"))
if message is None or message.get("content") == text:
return choice
return {**choice_obj, "message": {**message, "content": text}}
if message is not None:
return choice if message.get("content") == text else {**choice_obj, "message": {**message, "content": text}}
if isinstance(choice_obj.get("text"), str):
return {**choice_obj, "text": text}
return choice

17
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import partition_annotations
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
)
@ -1700,10 +1701,17 @@ def stream_chunk_builder(
# Merge annotations from ALL chunks — providers may spread
# them across multiple streaming chunks or send them only in
# the final chunk.
all_annotations: Final[list] = []
all_annotations: Final[list[object]] = []
for ac in annotation_chunks:
all_annotations.extend(ac["choices"][0]["delta"]["annotations"])
response["choices"][0]["message"]["annotations"] = all_annotations
message_annotations, provider_annotations = partition_annotations(all_annotations)
if message_annotations is not None:
response["choices"][0]["message"]["annotations"] = message_annotations
if provider_annotations:
response.choices[0].message.provider_specific_fields = {
**(response.choices[0].message.provider_specific_fields or {}),
"annotations": list(provider_annotations),
}
audio_chunks: Final = [
chunk
@ -1766,7 +1774,10 @@ def stream_chunk_builder(
if combined_provider_fields:
_choice = response.choices[0]
_choice.message.provider_specific_fields = combined_provider_fields
_choice.message.provider_specific_fields = {
**(_choice.message.provider_specific_fields or {}),
**combined_provider_fields,
}
completion_output = get_content_from_model_response(response)

31
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -123,6 +123,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
from litellm.litellm_core_utils.prompt_templates.common_utils import (
anthropic_image_source_to_openai_url,
anthropic_inline_document,
parse_tool_call_arguments,
reasoning_content_from_thinking_blocks,
with_prompt_cache_breakpoint,
@ -186,6 +187,8 @@ from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
ChatCompletionFileObject,
ChatCompletionFileObjectFile,
ChatCompletionImageObject,
ChatCompletionImageUrlObject,
ChatCompletionRedactedThinkingBlock,
@ -500,7 +503,9 @@ class LiteLLMAnthropicMessagesAdapter:
for m in ordered_messages:
user_message: ChatCompletionUserMessage | None = None
tool_message_list: list[ChatCompletionToolMessage] = []
new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = []
new_user_content_list: list[
ChatCompletionTextObject | ChatCompletionImageObject | ChatCompletionFileObject
] = []
if m["role"] == "system":
system_message = self._translate_midturn_system_message_to_openai(m, model)
if system_message is not None:
@ -533,13 +538,8 @@ class LiteLLMAnthropicMessagesAdapter:
self._add_prompt_cache_breakpoint_if_present(content, image_obj)
)
elif content.get("type") == "document":
# Convert Anthropic document format (PDF, etc.) to OpenAI format
source = content.get("source", {})
openai_image_url = self._translate_anthropic_image_to_openai(cast(dict, source))
if openai_image_url:
image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url)
doc_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj)
doc_obj = self._translate_anthropic_document_to_openai(content)
if doc_obj is not None:
self._add_cache_control_if_applicable(content, doc_obj, model)
new_user_content_list.append(doc_obj)
elif content.get("type") == "tool_result":
@ -1369,7 +1369,7 @@ class LiteLLMAnthropicMessagesAdapter:
text: Final = tool_result_text_part(block)
return None if text is None else ChatCompletionTextObject(type="text", text=text)
case "image" | "document":
return self._tool_result_image_part(block.get("source"))
return self._source_image_url_part(block.get("source"))
case "tool_reference":
return ChatCompletionToolReferenceObject(
type="tool_reference", tool_name=str(block.get("tool_name") or "")
@ -1377,7 +1377,18 @@ class LiteLLMAnthropicMessagesAdapter:
case _:
return None
def _tool_result_image_part(self, image_source: object) -> ChatCompletionImageObject | None:
def _translate_anthropic_document_to_openai(
self, document: Mapping[str, object]
) -> ChatCompletionFileObject | ChatCompletionImageObject | None:
inline_document: Final = anthropic_inline_document(document)
if inline_document is None:
return self._source_image_url_part(document.get("source"))
return ChatCompletionFileObject(
type="file",
file=ChatCompletionFileObjectFile(filename=inline_document.filename, file_data=inline_document.file_data),
)
def _source_image_url_part(self, image_source: object) -> ChatCompletionImageObject | None:
if not isinstance(image_source, dict):
return None
openai_image_url = self._translate_anthropic_image_to_openai(image_source)

26
litellm/llms/anthropic/messages/responses_adapters/transformation.py
View file

@ -13,6 +13,7 @@ from typing import Any, Final, cast
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
anthropic_inline_document,
encrypted_reasoning_signature,
responses_reasoning_items_from_thinking_blocks,
with_prompt_cache_breakpoint,
@ -106,27 +107,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
block: Mapping[str, object],
) -> dict[str, str] | None: # mutable-ok: API message payload
"""Convert an Anthropic document block to a Responses input_file part."""
inline_document: Final = anthropic_inline_document(block)
if inline_document is not None:
return {
"type": "input_file",
"filename": inline_document.filename,
"file_data": inline_document.file_data,
}
raw_source: Final = block.get("source")
if not isinstance(raw_source, Mapping):
return None
source: Final = cast(Mapping[str, object], raw_source) # cast-ok: untrusted client payload
source_type: Final = source.get("type")
if source_type == "base64":
data: Final = source.get("data")
if not isinstance(data, str) or not data:
return None
raw_media_type: Final = source.get("media_type")
media_type: Final = (
raw_media_type if isinstance(raw_media_type, str) and raw_media_type else "application/pdf"
)
raw_title: Final = block.get("title")
filename: Final = raw_title if isinstance(raw_title, str) and raw_title else "document.pdf"
return {
"type": "input_file",
"filename": filename,
"file_data": f"data:{media_type};base64,{data}",
}
if source_type == "url":
if source.get("type") == "url":
url: Final = source.get("url")
if not isinstance(url, str) or not url:
return None

22
litellm/llms/azure_ai/chat/transformation.py
View file

@ -11,10 +11,11 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
audio_or_image_in_message_content,
content_parts,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
is_pdf_file_part,
)
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -56,6 +57,20 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
"cache_control",
)
LIST_ONLY_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_audio"})
def _is_list_only_content_part(part: object) -> bool:
match part:
case {"type": str() as part_type} if part_type in LIST_ONLY_CONTENT_PART_TYPES:
return True
case _:
return is_pdf_file_part(part)
def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
return any(_is_list_only_content_part(part) for part in content_parts(message))
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
@ -256,7 +271,7 @@ class AzureAIStudioConfig(OpenAIConfig):
these with "Extra inputs are not permitted", which breaks multi-turn
Anthropic-format clients that echo thinking blocks back as history.
2. Transforms list content to a string.
3. If message contains an image or audio, send as is (user-intended)
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
@ -267,8 +282,7 @@ class AzureAIStudioConfig(OpenAIConfig):
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
filter_value_from_dict(message_dict, field)
# Do nothing if the message contains an image or audio
if audio_or_image_in_message_content(message):
if _has_list_only_content_part(message):
continue
texts = convert_content_list_to_str(message=message)

9
litellm/llms/base_llm/realtime/transformation.py
View file

@ -4,7 +4,6 @@ from types import TracebackType
from typing import TYPE_CHECKING, Any, Protocol
import httpx
from typing_extensions import Self
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
from litellm.types.realtime import (
@ -24,7 +23,7 @@ else:
class RealtimeBackend(Protocol):
async def __aenter__(self) -> Self: ...
async def __aenter__(self) -> object: ...
async def __aexit__(
self,
@ -97,9 +96,15 @@ class BaseRealtimeConfig(ABC):
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
return None
def session_refusal(self) -> Exception | None:
return None
async def open_backend(self, url: str, headers: Mapping[str, str]) -> RealtimeBackend | None:
return None
def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend:
return backend
def transform_session_created_event(
self,
model: str,

3
litellm/llms/base_llm/vector_store/transformation.py
View file

@ -226,6 +226,9 @@ class BaseVectorStoreConfig:
``get_complete_url`` adds a query string, like Azure, override this."""
return f"{api_base}/{encoded_vector_store_id}"
def get_vector_store_list_url(self, api_base: str) -> str:
return api_base
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
from ..chat.transformation import BaseLLMException

10
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -6923,7 +6923,7 @@ class BaseLLMHTTPHandler:
try:
ssl_context: Final = realtime_ssl_for_url(url)
provider_backend: Final = await provider_config.open_backend(url, headers)
backend_ws: Final = (
backend_ws: Final = provider_config.wrap_backend(
provider_backend
if provider_backend is not None
else await self._open_realtime_backend_ws(websockets, url, headers, ssl_context)
@ -6961,9 +6961,7 @@ class BaseLLMHTTPHandler:
await backend_ws.send(_session_config)
realtime_streaming.session_configuration_request = _session_config
# For providers that defer setup until client session.update, optionally
# send synthetic session.created to unblock clients waiting on connect.
if not provider_config.requires_session_configuration():
if not _session_config:
synthetic_session: Final = provider_config.transform_session_created_event(
model=model,
logging_session_id=logging_obj.litellm_trace_id,
@ -11152,7 +11150,7 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url: Final = api_base
url: Final = vector_store_provider_config.get_vector_store_list_url(api_base)
params: Final[dict[str, object]] = {}
if after is not None:
@ -11231,7 +11229,7 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url: Final = api_base
url: Final = vector_store_provider_config.get_vector_store_list_url(api_base)
params: Final[dict[str, object]] = {}
if after is not None:

9
litellm/llms/dashscope/chat/transformation.py
View file

@ -10,6 +10,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ..common_utils import resolve_dashscope_openai_api_base
class DashScopeChatConfig(OpenAIGPTConfig):
@ -55,11 +56,9 @@ class DashScopeChatConfig(OpenAIGPTConfig):
def _get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
api_base = (
api_base or get_secret_str("DASHSCOPE_API_BASE") or "https://dashscope.aliyuncs.com/compatible-mode/v1"
)
resolved_api_base: Final = resolve_dashscope_openai_api_base(api_base)
dynamic_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY")
return api_base, dynamic_api_key
return resolved_api_base, dynamic_api_key
def get_openai_compatible_provider_info(
self,
@ -69,7 +68,7 @@ class DashScopeChatConfig(OpenAIGPTConfig):
return self._get_openai_compatible_provider_info(api_base, api_key)
def _resolve_chat_api_base(self, api_base: str | None) -> str:
return api_base or "https://dashscope.aliyuncs.com/compatible-mode/v1"
return resolve_dashscope_openai_api_base(api_base)
def get_complete_url(
self,

136
litellm/llms/dashscope/common_utils.py
View file

@ -2,6 +2,7 @@
Common utilities for the DashScope LLM provider.
"""
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from urllib.parse import urlparse
@ -19,10 +20,123 @@ if TYPE_CHECKING:
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
DASHSCOPE_CHAT_COMPATIBLE_PATH: Final = "/compatible-mode/v1"
DASHSCOPE_RERANK_PATH: Final = "/compatible-api/v1/reranks"
DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic"
DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1"
DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1"
DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks"
DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com"
_STANDARD_DASHSCOPE_HOSTS: Final = frozenset(
{
"dashscope.aliyuncs.com",
"dashscope-intl.aliyuncs.com",
"dashscope-us.aliyuncs.com",
}
)
_WORKSPACE_HOST_SUFFIX: Final = ".maas.aliyuncs.com"
_EXCLUDED_WORKSPACE_PREFIXES: Final = ("trial.", "token-plan.")
_PROTOCOL_PATH_PREFIXES: Final = (
DASHSCOPE_CHAT_COMPATIBLE_PATH,
DASHSCOPE_ANTHROPIC_PATH,
DASHSCOPE_NATIVE_V1_PATH,
DASHSCOPE_COMPATIBLE_API_PATH,
)
def _rerank_base_for_chat_shaped_base(api_base: str | None) -> str | None:
@dataclass(frozen=True, slots=True)
class DashScopeEndpoints:
origin: str
openai: str
anthropic: str
native: str
def _is_dashscope_host(host: str) -> bool:
if host in _STANDARD_DASHSCOPE_HOSTS:
return True
return host.endswith(_WORKSPACE_HOST_SUFFIX) and not host.startswith(_EXCLUDED_WORKSPACE_PREFIXES)
def _dashscope_origin(api_base: str | None) -> str | None:
if not api_base:
return None
parsed: Final = urlparse(api_base)
if parsed.scheme not in ("http", "https") or not parsed.netloc:
return None
if parsed.query or parsed.fragment:
return None
host: Final = (parsed.hostname or "").lower()
if not _is_dashscope_host(host):
return None
path: Final = parsed.path.rstrip("/")
if path and not any(path == prefix or path.startswith(f"{prefix}/") for prefix in _PROTOCOL_PATH_PREFIXES):
return None
return f"{parsed.scheme}://{parsed.netloc}"
def resolve_dashscope_endpoints(api_base: str | None) -> DashScopeEndpoints | None:
"""
Derive DashScope's protocol-specific bases from any standard DashScope base.
Accepts the domain root, an OpenAI-compatible base, an Anthropic Messages
base, or a native `/api/v1` base for `dashscope.aliyuncs.com`,
`dashscope-intl.aliyuncs.com`, `dashscope-us.aliyuncs.com`, and
`{workspace}.{region}.maas.aliyuncs.com`. Returns None for any other host
or path so callers keep the caller-supplied base untouched.
"""
origin: Final = _dashscope_origin(api_base)
if origin is None:
return None
return DashScopeEndpoints(
origin=origin,
openai=f"{origin}{DASHSCOPE_CHAT_COMPATIBLE_PATH}",
anthropic=f"{origin}{DASHSCOPE_ANTHROPIC_PATH}",
native=f"{origin}{DASHSCOPE_NATIVE_V1_PATH}",
)
def is_dashscope_native_endpoint_url(api_base: str) -> bool:
return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/")
def resolve_dashscope_openai_api_base(
api_base: str | None,
env_var: str = "DASHSCOPE_API_BASE",
default: str = f"{DASHSCOPE_DEFAULT_ORIGIN}{DASHSCOPE_CHAT_COMPATIBLE_PATH}",
) -> str:
candidate: Final = api_base or get_secret_str(env_var)
if candidate and is_dashscope_native_endpoint_url(candidate):
return candidate
endpoints: Final = resolve_dashscope_endpoints(candidate)
if endpoints is not None:
return endpoints.openai
return candidate or default
def resolve_dashscope_anthropic_api_base(
api_base: str | None,
env_var: str = "DASHSCOPE_API_BASE",
default: str = f"{DASHSCOPE_DEFAULT_ORIGIN}{DASHSCOPE_ANTHROPIC_PATH}",
) -> str:
candidate: Final = api_base or get_secret_str(env_var)
endpoints: Final = resolve_dashscope_endpoints(candidate)
if endpoints is not None:
return endpoints.anthropic
return candidate or default
def dashscope_base_selects_anthropic_messages(api_base: str | None) -> bool:
candidate: Final = api_base or get_secret_str("DASHSCOPE_API_BASE")
if not candidate:
return False
path: Final = urlparse(candidate).path.rstrip("/")
if path.endswith(DASHSCOPE_ANTHROPIC_PATH) or f"{DASHSCOPE_ANTHROPIC_PATH}/" in path:
return True
return not path and resolve_dashscope_endpoints(candidate) is not None
def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None:
if api_base is None:
return None
parsed: Final = urlparse(api_base)
@ -30,14 +144,24 @@ def _rerank_base_for_chat_shaped_base(api_base: str | None) -> str | None:
on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com")
if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
return None
return f"{parsed.scheme}://{parsed.netloc}{DASHSCOPE_RERANK_PATH}"
return f"{parsed.scheme}://{parsed.netloc}"
def _is_compatible_api_base(api_base: str) -> bool:
path: Final = urlparse(api_base).path.rstrip("/")
return path == DASHSCOPE_COMPATIBLE_API_PATH or path.startswith(f"{DASHSCOPE_COMPATIBLE_API_PATH}/")
def resolve_dashscope_family_rerank_api_base(api_base: str | None, env_var: str, default_rerank_base: str) -> str:
remapped: Final = _rerank_base_for_chat_shaped_base(api_base)
if api_base is not None and remapped is None:
if api_base is None:
return get_secret_str(env_var) or default_rerank_base
if _is_compatible_api_base(api_base):
return api_base
return get_secret_str(env_var) or remapped or default_rerank_base
chat_origin: Final = _chat_shaped_aliyun_origin(api_base)
if chat_origin is not None:
return get_secret_str(env_var) or f"{chat_origin}{DASHSCOPE_RERANK_PATH}"
endpoints: Final = resolve_dashscope_endpoints(api_base)
return f"{endpoints.origin}{DASHSCOPE_RERANK_PATH}" if endpoints is not None else api_base
def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbeddingConfig":

4
litellm/llms/dashscope/embed/transformation.py
View file

@ -25,7 +25,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, Usage
from ..common_utils import DashScopeError
from ..common_utils import DashScopeError, resolve_dashscope_openai_api_base
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
@ -91,7 +91,7 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
return resolved_api_key
def _resolve_embedding_api_base(self, api_base: str | None) -> str:
return api_base or get_secret_str("DASHSCOPE_API_BASE") or DEFAULT_API_BASE
return resolve_dashscope_openai_api_base(api_base, default=DEFAULT_API_BASE)
def validate_environment(
self,

35
litellm/llms/dashscope/image_generation/transformation.py
View file

@ -39,6 +39,13 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import ImageObject, ImageResponse
from ..common_utils import (
DASHSCOPE_CHAT_COMPATIBLE_PATH,
DASHSCOPE_IMAGE_PATH,
is_dashscope_native_endpoint_url,
resolve_dashscope_endpoints,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
@ -53,8 +60,6 @@ _DECODED_JSON: Final = TypeAdapter(object)
DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
CHAT_COMPATIBLE_MODE_PATH: Final = "/compatible-mode/v1"
# Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
"256x256": "256*256",
@ -102,8 +107,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
raise ValueError("DASHSCOPE_API_KEY is not set")
return resolved_api_key
def _resolve_image_api_base(self, image_api_base: str | None) -> str:
return image_api_base or get_secret_str("DASHSCOPE_API_BASE_IMAGE") or DEFAULT_API_BASE
def _image_api_base_env(self) -> str | None:
return get_secret_str("DASHSCOPE_API_BASE_IMAGE")
def _default_image_api_base(self) -> str:
return DEFAULT_API_BASE
def _configured_chat_api_base(self) -> str | None:
return get_secret_str("DASHSCOPE_API_BASE")
def get_complete_url(
self,
@ -114,10 +125,18 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
litellm_params: dict,
stream: bool | None = None,
) -> str:
image_api_base: Final = (
api_base if api_base and not api_base.rstrip("/").endswith(CHAT_COMPATIBLE_MODE_PATH) else None
)
return self._resolve_image_api_base(image_api_base)
image_env_api_base: Final = self._image_api_base_env()
configured_api_base: Final = api_base or self._configured_chat_api_base()
if not configured_api_base or configured_api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH):
return image_env_api_base or self._default_image_api_base()
if is_dashscope_native_endpoint_url(configured_api_base):
return configured_api_base
endpoints: Final = resolve_dashscope_endpoints(configured_api_base)
if endpoints is None:
return configured_api_base
if not api_base and image_env_api_base:
return image_env_api_base
return f"{endpoints.origin}{DASHSCOPE_IMAGE_PATH}"
def validate_environment(
self,

7
litellm/llms/dashscope/messages/__init__.py Normal file
View file

@ -0,0 +1,7 @@
"""
DashScope Anthropic-compatible Messages Module
"""
from .transformation import DashScopeAnthropicMessagesConfig
__all__ = ["DashScopeAnthropicMessagesConfig"]

96
litellm/llms/dashscope/messages/transformation.py Normal file
View file

@ -0,0 +1,96 @@
"""
DashScope Anthropic-compatible Messages transformation config.
"""
from typing import ClassVar, Final
import litellm
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
from ..common_utils import (
dashscope_base_selects_anthropic_messages,
resolve_dashscope_anthropic_api_base,
)
class DashScopeAnthropicMessagesConfig(AnthropicMessagesConfig):
"""
DashScope exposes an Anthropic-compatible Messages API at
``{origin}/apps/anthropic`` for every DashScope domain, including
``dashscope.aliyuncs.com`` and ``{workspace}.{region}.maas.aliyuncs.com``.
"""
_workload_identity_eligible: ClassVar[bool] = False
@property
def custom_llm_provider(self) -> str | None:
return "dashscope"
def should_strip_billing_metadata(self) -> bool:
return True
def handles_web_search_natively(self) -> bool:
return False
def serves_api_base(self, model: str, api_base: str | None) -> bool:
return dashscope_base_selects_anthropic_messages(api_base)
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return (
normalize_nonempty_secret_str(api_key)
or normalize_nonempty_secret_str(get_secret_str("DASHSCOPE_API_KEY"))
or normalize_nonempty_secret_str(litellm.api_key)
)
@staticmethod
def get_api_base(api_base: str | None = None) -> str:
return resolve_dashscope_anthropic_api_base(api_base)
def validate_anthropic_messages_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors the legacy base override signature
model: str,
messages: list[object], # mutable-ok: mirrors the legacy base override signature
optional_params: dict[str, object], # mutable-ok: mirrors the legacy base override signature
litellm_params: dict[str, object], # mutable-ok: mirrors the legacy base override signature
api_key: str | None = None,
api_base: str | None = None,
) -> tuple[dict[str, str], str | None]: # mutable-ok: mirrors the base signature
resolved_api_key: Final = self.get_api_key(api_key=api_key)
if resolved_api_key is None:
present_headers: Final = {name.lower() for name in headers}
if "x-api-key" not in present_headers and "authorization" not in present_headers:
raise AuthenticationError(
message=(
"Missing DashScope API key. Set DASHSCOPE_API_KEY or pass api_key explicitly. "
"Anthropic credentials are never sent to a DashScope endpoint."
),
llm_provider="dashscope",
model=model,
)
return super().validate_anthropic_messages_environment( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportReturnType] # base signature uses an unparameterized dict
headers=headers,
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
api_key=resolved_api_key,
api_base=api_base,
)
def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict[str, object], # mutable-ok: mirrors the base get_complete_url signature
litellm_params: dict[str, object], # mutable-ok: mirrors the base get_complete_url signature
stream: bool | None = None,
) -> str:
return ensure_api_route(self.get_api_base(api_base=api_base), "v1/messages")

16
litellm/llms/dashscope/qwen_ai_platform.py
View file

@ -3,7 +3,7 @@ from typing import Final
from litellm.secret_managers.main import get_secret_str
from .chat.transformation import DashScopeChatConfig
from .common_utils import resolve_dashscope_family_rerank_api_base
from .common_utils import resolve_dashscope_family_rerank_api_base, resolve_dashscope_openai_api_base
from .embed.transformation import DashScopeEmbeddingConfig
from .image_generation.transformation import DashScopeImageGenerationConfig
from .rerank.transformation import DashScopeRerankConfig
@ -38,7 +38,7 @@ class QwenAIPlatformChatConfig(DashScopeChatConfig):
return self._resolve_chat_api_base(api_base), _resolve_qwen_ai_platform_api_key(api_key)
def _resolve_chat_api_base(self, api_base: str | None) -> str:
return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE") or QWEN_AI_PLATFORM_API_BASE
return resolve_dashscope_openai_api_base(api_base, "QWEN_AI_PLATFORM_API_BASE", QWEN_AI_PLATFORM_API_BASE)
class QwenAIPlatformEmbeddingConfig(DashScopeEmbeddingConfig):
@ -46,7 +46,7 @@ class QwenAIPlatformEmbeddingConfig(DashScopeEmbeddingConfig):
return _require_qwen_ai_platform_api_key(api_key)
def _resolve_embedding_api_base(self, api_base: str | None) -> str:
return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE") or QWEN_AI_PLATFORM_API_BASE
return resolve_dashscope_openai_api_base(api_base, "QWEN_AI_PLATFORM_API_BASE", QWEN_AI_PLATFORM_API_BASE)
class QwenAIPlatformRerankConfig(DashScopeRerankConfig):
@ -63,8 +63,14 @@ class QwenAIPlatformImageGenerationConfig(DashScopeImageGenerationConfig):
def _resolve_api_key(self, api_key: str | None) -> str:
return _require_qwen_ai_platform_api_key(api_key)
def _resolve_image_api_base(self, image_api_base: str | None) -> str:
return image_api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_IMAGE") or QWEN_AI_PLATFORM_IMAGE_API_BASE
def _image_api_base_env(self) -> str | None:
return get_secret_str("QWEN_AI_PLATFORM_API_BASE_IMAGE")
def _default_image_api_base(self) -> str:
return QWEN_AI_PLATFORM_IMAGE_API_BASE
def _configured_chat_api_base(self) -> str | None:
return get_secret_str("QWEN_AI_PLATFORM_API_BASE")
class QwenAIPlatformVideoConfig(DashScopeVideoConfig):

16
litellm/llms/dashscope/qwencloud.py
View file

@ -3,7 +3,7 @@ from typing import Final
from litellm.secret_managers.main import get_secret_str
from .chat.transformation import DashScopeChatConfig
from .common_utils import resolve_dashscope_family_rerank_api_base
from .common_utils import resolve_dashscope_family_rerank_api_base, resolve_dashscope_openai_api_base
from .embed.transformation import DashScopeEmbeddingConfig
from .image_generation.transformation import DashScopeImageGenerationConfig
from .rerank.transformation import DashScopeRerankConfig
@ -38,7 +38,7 @@ class QwenCloudChatConfig(DashScopeChatConfig):
return self._resolve_chat_api_base(api_base), _resolve_qwencloud_api_key(api_key)
def _resolve_chat_api_base(self, api_base: str | None) -> str:
return api_base or get_secret_str("QWENCLOUD_API_BASE") or QWENCLOUD_API_BASE
return resolve_dashscope_openai_api_base(api_base, "QWENCLOUD_API_BASE", QWENCLOUD_API_BASE)
class QwenCloudEmbeddingConfig(DashScopeEmbeddingConfig):
@ -46,7 +46,7 @@ class QwenCloudEmbeddingConfig(DashScopeEmbeddingConfig):
return _require_qwencloud_api_key(api_key)
def _resolve_embedding_api_base(self, api_base: str | None) -> str:
return api_base or get_secret_str("QWENCLOUD_API_BASE") or QWENCLOUD_API_BASE
return resolve_dashscope_openai_api_base(api_base, "QWENCLOUD_API_BASE", QWENCLOUD_API_BASE)
class QwenCloudRerankConfig(DashScopeRerankConfig):
@ -63,8 +63,14 @@ class QwenCloudImageGenerationConfig(DashScopeImageGenerationConfig):
def _resolve_api_key(self, api_key: str | None) -> str:
return _require_qwencloud_api_key(api_key)
def _resolve_image_api_base(self, image_api_base: str | None) -> str:
return image_api_base or get_secret_str("QWENCLOUD_API_BASE_IMAGE") or QWENCLOUD_IMAGE_API_BASE
def _image_api_base_env(self) -> str | None:
return get_secret_str("QWENCLOUD_API_BASE_IMAGE")
def _default_image_api_base(self) -> str:
return QWENCLOUD_IMAGE_API_BASE
def _configured_chat_api_base(self) -> str | None:
return get_secret_str("QWENCLOUD_API_BASE")
class QwenCloudVideoConfig(DashScopeVideoConfig):

5
litellm/llms/dashscope/videos/transformation.py
View file

@ -38,6 +38,8 @@ from litellm.types.videos.utils import (
)
from litellm.utils import get_model_info
from ..common_utils import resolve_dashscope_endpoints
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -206,6 +208,9 @@ def _parse_task_response(raw_response: httpx.Response) -> _TaskResponse:
def _normalized_api_base(api_base: str) -> str:
endpoints: Final = resolve_dashscope_endpoints(api_base)
if endpoints is not None:
return endpoints.origin
trimmed: Final = api_base.rstrip("/")
if trimmed.endswith(DASHSCOPE_COMPATIBLE_MODE_PATH):
return trimmed[: -len(DASHSCOPE_COMPATIBLE_MODE_PATH)]

15
litellm/llms/gigachat/chat/transformation.py
View file

@ -38,6 +38,15 @@ else:
_EMPTY_FUNCTION: Final[Mapping[str, object]] = MappingProxyType({})
_JSON_VALUE: Final = TypeAdapter(object)
_CONTENT_PART: Final = TypeAdapter(Mapping[str, object])
def _file_part_data(part: Mapping[str, object]) -> str | None:
match part:
case {"type": "file", "file": {"file_data": str() as file_data}} if file_data:
return file_data
case _:
return None
def is_valid_json(value: str) -> bool:
@ -295,7 +304,7 @@ class GigaChatConfig(BaseConfig):
def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[object]]:
"""
Extract text and image attachments from a multimodal message content list.
Extract text, image and file attachments from a multimodal message content list.
Args:
content: List of content parts (OpenAI multimodal format)
@ -320,6 +329,10 @@ class GigaChatConfig(BaseConfig):
file_id = self._upload_image(upload_url)
if file_id:
attachments.append(file_id)
elif (file_data := _file_part_data(_CONTENT_PART.validate_python(part))) and (
uploaded_file_id := self._upload_image(file_data)
):
attachments.append(uploaded_file_id)
text: Final = "\n".join(texts) if texts else ""
return text, attachments

5
litellm/llms/oci/chat/transformation.py
View file

@ -20,6 +20,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.url_utils import join_api_base
@ -697,7 +698,9 @@ class OCIChatConfig(BaseConfig):
servingMode=serving_mode,
chatRequest=OCIChatRequestPayload(
apiFormat=vendor.value,
messages=adapt_messages_to_generic_oci_standard(messages),
messages=adapt_messages_to_generic_oci_standard(
list(messages_with_file_parts_as_image_urls(messages))
),
**self._get_optional_params(vendor, optional_params, model),
),
)

198
litellm/llms/openai/completion/guardrail_translation/handler.py
View file

@ -5,18 +5,30 @@ This module provides guardrail translation support for OpenAI's text completion
The handler processes the 'prompt' parameter for guardrails.
"""
from collections.abc import Mapping
import json
from collections.abc import Mapping, Sequence
from itertools import chain
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.utils import GenericGuardrailAPIInputs
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
StreamTransformSink,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
blocked_chat_stream_usage,
stream_item_field,
stream_item_items,
)
from litellm.types.utils import GenericGuardrailAPIInputs, TextChoices, TextCompletionResponse, Usage
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.types.utils import TextCompletionResponse
class OpenAITextCompletionHandler(BaseTranslation):
@ -32,6 +44,9 @@ class OpenAITextCompletionHandler(BaseTranslation):
- A list of strings (for batch completions)
"""
delivers_ended_stream_rewrites = True
assembles_streamed_response = True
async def process_input_messages(
self,
data: dict,
@ -125,7 +140,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
user_api_key_dict: "UserAPIKeyAuth | None" = None,
request_data: dict | None = None,
request_data: dict[str, object] | None = None,
) -> "TextCompletionResponse":
"""
Process output response by applying guardrails to completion text.
@ -162,11 +177,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
if "response" not in request_data:
request_data["response"] = response
# Add user API key metadata with prefixed keys
if "litellm_metadata" not in request_data:
user_metadata: Final = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
if user_metadata:
request_data["litellm_metadata"] = user_metadata
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
# Include model information from the response if available
@ -195,3 +206,168 @@ class OpenAITextCompletionHandler(BaseTranslation):
)
return response
async def process_output_streaming_response(
self,
responses_so_far: list[object], # mutable-ok: rewrites the caller's buffered chunks in place
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
user_api_key_dict: "UserAPIKeyAuth | None" = None,
request_data: dict[str, object] | None = None, # mutable-ok: the hooks share and mutate the request payload
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> Sequence[object]:
chunks: Final = _text_completion_chunks(responses_so_far)
if not self._check_streaming_has_ended(chunks):
return responses_so_far
assembled: Final = self._assemble_ended_stream(chunks)
pre_guardrail_texts: Final = _choice_texts(assembled)
await self.process_output_response(
response=assembled,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
)
if not deliver_ended_stream_rewrites:
return responses_so_far
rewrites: Final = tuple(
after for before, after in zip(pre_guardrail_texts, _choice_texts(assembled)) if after != before
)
for choice_index, text in rewrites:
self._write_choice_text(chunks, choice_index, text)
return responses_so_far
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
chunks: Final = _text_completion_chunks(responses_so_far)
if not self._check_streaming_has_ended(chunks):
return None
assembled: Final = self._assemble_ended_stream(chunks)
return StreamingScanKey(texts=tuple(text for _, text in _choice_texts(assembled)), stream_ended=True)
def released_stream_as_ended(self, responses_so_far: Sequence[object]) -> tuple[object, ...]:
chunks: Final = _text_completion_chunks(responses_so_far)
streamed_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks))
finished_indices: Final = frozenset(
_choice_index(choice) for choice in streamed_choices if choice.finish_reason
)
unfinished_indices: Final = tuple(
dict.fromkeys(
_choice_index(choice) for choice in streamed_choices if _choice_index(choice) not in finished_indices
)
)
if not unfinished_indices:
return tuple(responses_so_far)
terminator: Final = TextCompletionResponse(
id=chunks[0].id,
model=chunks[0].model,
object="text_completion",
choices=[TextChoices(index=index, text="", finish_reason="length") for index in unfinished_indices],
)
return (*responses_so_far, terminator)
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes]:
streamed: Final = _text_completion_chunks(responses_so_far or ())
source: Final = streamed[0] if streamed else TextCompletionResponse(model=exc.model)
message_chunk: Final = TextCompletionResponse(
id=source.id,
created=source.created,
model=source.model or exc.model,
object="text_completion",
choices=[TextChoices(index=0, text=exc.message)],
).model_copy(update={"usage": None})
final_chunk: Final = TextCompletionResponse(
id=source.id,
created=source.created,
model=source.model or exc.model,
object="text_completion",
choices=[TextChoices(index=0, text="", finish_reason="content_filter")],
).model_copy(update={"usage": _blocked_stream_usage(exc.original_response)})
return tuple(
f"data: {chunk.model_dump_json(exclude_none=True)}\n\n".encode() for chunk in (message_chunk, final_chunk)
)
def build_stream_error_items(
self,
exc: "HTTPException",
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes] | None:
from litellm.proxy.common_request_processing import sse_error_payload
_, error_obj = sse_error_payload(exc)
return (f'data: {{"error": {json.dumps(error_obj)}}}\n\n'.encode(),)
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
streamed_choices: Final = chain.from_iterable(stream_item_items(item, "choices") for item in responses_so_far)
return any(stream_item_field(choice, "finish_reason") is not None for choice in streamed_choices)
@staticmethod
def _assemble_ended_stream(responses_so_far: Sequence[TextCompletionResponse]) -> TextCompletionResponse:
streamed_choices: Final = tuple(chain.from_iterable(item.choices for item in responses_so_far))
choice_indices: Final = tuple(dict.fromkeys(_choice_index(choice) for choice in streamed_choices))
assembled_choices: Final = [
TextChoices(
index=index,
text="".join(_choice_text(choice) for choice in streamed_choices if _choice_index(choice) == index),
finish_reason=next(
(
choice.finish_reason
for choice in streamed_choices
if _choice_index(choice) == index and choice.finish_reason
),
None,
),
)
for index in choice_indices
]
first: Final = responses_so_far[0]
return TextCompletionResponse(
id=first.id,
created=first.created,
model=first.model,
object="text_completion",
choices=assembled_choices,
usage=next((item.usage for item in reversed(responses_so_far) if item.usage is not None), None),
)
@staticmethod
def _write_choice_text(responses_so_far: Sequence[TextCompletionResponse], choice_index: int, text: str) -> None:
streamed_choices: Final = chain.from_iterable(item.choices for item in responses_so_far)
head, *tail = (choice for choice in streamed_choices if _choice_index(choice) == choice_index)
head.text = text
for choice in tail:
if isinstance(stream_item_field(choice, "text"), str):
choice.text = ""
def _blocked_stream_usage(original_response: object) -> Usage | None:
prompt_tokens, completion_tokens = blocked_chat_stream_usage(original_response)
if isinstance(original_response, list) and prompt_tokens == completion_tokens == 0:
return None
return Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
def _text_completion_chunks(responses_so_far: Sequence[object]) -> tuple[TextCompletionResponse, ...]:
return tuple(item for item in responses_so_far if isinstance(item, TextCompletionResponse))
def _choice_text(choice: TextChoices) -> str:
text: Final = stream_item_field(choice, "text")
return text if isinstance(text, str) else ""
def _choice_index(choice: TextChoices) -> int:
return choice.index or 0
def _choice_texts(response: TextCompletionResponse) -> tuple[tuple[int, str], ...]:
return tuple((_choice_index(choice), _choice_text(choice)) for choice in response.choices)

0
litellm/llms/opensearch/__init__.py Normal file
View file

0
litellm/llms/opensearch/vector_stores/__init__.py Normal file
View file

415
litellm/llms/opensearch/vector_stores/transformation.py Normal file
View file

@ -0,0 +1,415 @@
import hashlib
import json
import re
from collections.abc import Mapping, Sequence
from functools import partial
from typing import TYPE_CHECKING, Final, Literal, NoReturn
from urllib.parse import quote, urlsplit
import httpx
from pydantic import ConfigDict, Field, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.exceptions import AuthenticationError, BadRequestError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.vector_store.transformation import (
BaseQueryEmbeddingVectorStoreConfig,
VectorStoreEmbeddingExecutor,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, SignsRequestsWithAWS, sign_aws_json_post
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.bedrock import AwsAuthParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import EmbeddingResponse
from litellm.types.vector_stores import (
BaseVectorStoreAuthCredentials,
VectorStoreCreateOptionalRequestParams,
VectorStoreIndexEndpoints,
VectorStoreResultContent,
VectorStoreSearchOptionalRequestParams,
VectorStoreSearchResponse,
VectorStoreSearchResult,
)
if TYPE_CHECKING:
from botocore.credentials import Credentials
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
DEFAULT_EMBEDDING_FIELD: Final = "embedding"
DEFAULT_TEXT_FIELD: Final = "text"
DEFAULT_MAX_NUM_RESULTS: Final = 10
MAX_MAX_NUM_RESULTS: Final = 50
_PARAM_PREFIX: Final = "opensearch_"
_AWS_HOSTS: Final = (
re.compile(r"\.(?P<region>[a-z0-9-]+)\.(?P<service>aoss|aoss-fips|es)\.amazonaws\.com(?:\.cn)?\Z"),
re.compile(r"\.(?P<service>aoss|aos)\.(?P<region>[a-z0-9-]+)\.on\.aws\Z"),
)
_ACCOUNT_ID_LABEL: Final = re.compile(r"\A\d{12}\.")
_QUERY_VECTOR: Final = TypeAdapter(tuple[float, ...])
_ATTRIBUTE_VALUE_TYPES: Final = (str, int, float, bool)
_SEARCH_ONLY_MESSAGE: Final = (
"The OpenSearch vector store is search-only. Create the k-NN index in OpenSearch, "
"then register it here with its name as opensearch_index_name or vector_store_id."
)
_PER_ACCOUNT_MESSAGE: Final = (
"api_base is an OpenSearch Serverless per-account endpoint, which needs a collection header the vector store "
"does not send. Use the collection endpoint instead, for example https://<collection-id>.<region>.aoss.amazonaws.com."
)
_SIGV4_ONLY_MESSAGE: Final = (
"The OpenSearch vector store signs every request with AWS SigV4 and does not support api_key. "
"Remove api_key and give the vector store AWS credentials instead."
)
_CREDENTIALS_HINT: Final = (
"Set aws_access_key_id and aws_secret_access_key, aws_role_name, or aws_profile_name on the vector store, "
"or run LiteLLM with an AWS identity."
)
def _bad_request(message: str) -> BadRequestError:
return BadRequestError(message=message, model=None, llm_provider="opensearch")
def _unauthenticated(reason: str) -> AuthenticationError:
return AuthenticationError(
message=f"No usable AWS credentials to sign the OpenSearch request ({reason}). {_CREDENTIALS_HINT}",
llm_provider="opensearch",
model=None,
)
def _usable_credentials(credentials: object) -> "Credentials":
from botocore.credentials import Credentials
if isinstance(credentials, Credentials):
return credentials
raise _unauthenticated("none were found")
class _OpenSearchParams(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
litellm_embedding_model: str | None = None
opensearch_embedding_field: str | None = None
opensearch_index_name: str | None = None
opensearch_text_field: str | None = None
@property
def embedding_field(self) -> str:
return self.opensearch_embedding_field or DEFAULT_EMBEDDING_FIELD
@property
def text_field(self) -> str:
return self.opensearch_text_field or DEFAULT_TEXT_FIELD
def index_name(self, vector_store_id: str) -> str:
return self.opensearch_index_name or vector_store_id
def require_embedding_model(self) -> str:
if not self.litellm_embedding_model:
raise _bad_request(
"litellm_embedding_model is required for the OpenSearch vector store. It must be the model that "
f"produced the vectors in '{self.embedding_field}', for example openai/text-embedding-3-small."
)
return self.litellm_embedding_model
class _SigningRegion(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
aws_region_name: str | None = None
class _SearchOptions(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
max_num_results: int | None = None
filters: Mapping[str, object] | None = None
ranking_options: Mapping[str, object] | None = None
rewrite_query: bool | None = None
@property
def unsupported(self) -> tuple[str, ...]:
requested: Final = (
("filters", self.filters),
("ranking_options", self.ranking_options),
("rewrite_query", self.rewrite_query),
)
return tuple(name for name, value in requested if value)
@property
def k(self) -> int:
if self.max_num_results is None:
return DEFAULT_MAX_NUM_RESULTS
if not 1 <= self.max_num_results <= MAX_MAX_NUM_RESULTS:
raise _bad_request(
f"max_num_results must be between 1 and {MAX_MAX_NUM_RESULTS}, got {self.max_num_results}"
)
return self.max_num_results
_KNOWN_PARAMS: Final = frozenset(name for name in _OpenSearchParams.model_fields if name.startswith(_PARAM_PREFIX))
class _LoggedSearch(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
query: str | tuple[str, ...] = ""
litellm_params: _OpenSearchParams = _OpenSearchParams()
@property
def search_query(self) -> str:
return BaseQueryEmbeddingVectorStoreConfig.query_text(self.query)
class _Hit(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
id: str = Field(alias="_id")
score: float | None = Field(default=None, alias="_score")
source: Mapping[str, object] = Field(default_factory=dict, alias="_source")
class _Hits(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
hits: Sequence[_Hit]
class _SearchResponse(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
hits: _Hits
def aws_signing_target(url: str) -> tuple[Literal["aoss", "es"], str | None]:
hostname: Final = urlsplit(url).hostname or ""
match: Final = next(filter(None, (pattern.search(hostname) for pattern in _AWS_HOSTS)), None)
if match is None:
return "es", None
return ("aoss" if match["service"].startswith("aoss") else "es"), match["region"]
def _search_result(hit: _Hit, params: _OpenSearchParams) -> VectorStoreSearchResult:
text: Final = hit.source.get(params.text_field)
filename: Final = hit.source.get("filename")
return VectorStoreSearchResult(
score=hit.score,
content=[VectorStoreResultContent(type="text", text=text)] if isinstance(text, str) else [],
file_id=hit.id,
filename=filename if isinstance(filename, str) else None,
attributes={
key: value
for key, value in hit.source.items()
if key not in (params.text_field, params.embedding_field) and isinstance(value, _ATTRIBUTE_VALUE_TYPES)
},
)
class OpenSearchVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, SignsRequestsWithAWS):
def __init__(self, aws: BaseAWSLLM | None = None) -> None:
self._aws: Final = aws if aws is not None else BaseAWSLLM()
def get_auth_credentials(self, litellm_params: Mapping[str, object]) -> BaseVectorStoreAuthCredentials:
return BaseVectorStoreAuthCredentials()
def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
return VectorStoreIndexEndpoints(read=[], write=[])
def validate_environment(
self, headers: Mapping[str, object], litellm_params: GenericLiteLLMParams | None
) -> dict[str, object]: # mutable-ok: the shared HTTP handler requires writable headers
params: Final = dict(litellm_params or {})
self._signing_credentials(params, self._signing_region(params, str(params.get("api_base") or "")))
return {**headers, "Content-Type": "application/json"}
def _signing_region(self, params: Mapping[str, object], api_base: str) -> str:
host_region: Final = aws_signing_target(api_base)[1]
return self._aws.get_aws_region_name_for_non_llm_api_calls(
host_region or _SigningRegion.model_validate(params).aws_region_name
)
def _signing_credentials(self, params: Mapping[str, object], region: str) -> "Credentials":
if params.get("api_key"):
raise _bad_request(_SIGV4_ONLY_MESSAGE)
auth: Final = AwsAuthParams.model_validate(params)
if bool(auth.aws_access_key_id) != bool(auth.aws_secret_access_key):
raise _unauthenticated("aws_access_key_id and aws_secret_access_key must be set together")
ensure_optional_import("botocore")
from botocore.exceptions import (
CredentialRetrievalError,
NoCredentialsError,
PartialCredentialsError,
ProfileNotFound,
)
try:
return _usable_credentials(self._aws.resolve_credentials(auth, region))
except CredentialRetrievalError as error:
verbose_logger.warning("OpenSearch vector store AWS credential source failed: %s", error)
raise _unauthenticated("the AWS credential source failed, see the LiteLLM logs") from None
except (NoCredentialsError, PartialCredentialsError, ProfileNotFound) as error:
raise _unauthenticated(str(error)) from None
def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
parsed: Final = urlsplit(api_base or "")
if parsed.scheme not in ("http", "https") or not parsed.hostname:
raise _bad_request(
"api_base must be the OpenSearch endpoint URL, for example "
"https://<collection-id>.<region>.aoss.amazonaws.com or https://<domain-endpoint>."
)
if _ACCOUNT_ID_LABEL.match(parsed.hostname) and aws_signing_target(api_base or "")[0] == "aoss":
raise _bad_request(_PER_ACCOUNT_MESSAGE)
return (api_base or "").rstrip("/")
def get_vector_store_url(self, api_base: str, encoded_vector_store_id: str) -> NoReturn:
raise _bad_request(_SEARCH_ONLY_MESSAGE)
def get_vector_store_list_url(self, api_base: str) -> NoReturn:
raise _bad_request(_SEARCH_ONLY_MESSAGE)
@staticmethod
def _params(
litellm_params: Mapping[str, object],
optional_params: VectorStoreSearchOptionalRequestParams,
extra_body: Mapping[str, object] | None,
) -> tuple[_OpenSearchParams, int]:
unknown: Final = sorted(
key for key in litellm_params if key.startswith(_PARAM_PREFIX) and key not in _KNOWN_PARAMS
)
if unknown:
raise _bad_request(
f"Unknown OpenSearch vector store parameter(s): {', '.join(unknown)}. "
f"Supported: {', '.join(sorted(_KNOWN_PARAMS))}."
)
if extra_body:
raise _bad_request("The OpenSearch vector store does not support extra_body.")
try:
options: Final = _SearchOptions.model_validate(optional_params)
except ValidationError as error:
raise _bad_request(f"Invalid vector store search parameters: {error}") from None
if options.unsupported and not drops_unsupported_params(litellm_params.get("drop_params")):
raise _bad_request(
f"The OpenSearch vector store does not support {', '.join(options.unsupported)} yet. "
"Remove it from the request, or set drop_params to drop it."
)
try:
return _OpenSearchParams.model_validate(litellm_params), options.k
except ValidationError:
raise _bad_request(
"opensearch_embedding_field, opensearch_index_name, opensearch_text_field, and litellm_embedding_model "
"must be strings."
) from None
@staticmethod
def _search_request(
vector_store_id: str,
embedding: EmbeddingResponse,
k: int,
params: _OpenSearchParams,
api_base: str,
) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body
vector: Final = _QUERY_VECTOR.validate_python(embedding.data[0]["embedding"]) # pyright: ignore[reportUnknownMemberType] # EmbeddingResponse.data is an untyped list, the vector is validated here
return (
f"{api_base}/{quote(params.index_name(vector_store_id), safe='')}/_search",
{
"size": k,
"query": {"knn": {params.embedding_field: {"vector": vector, "k": k}}},
"_source": {"excludes": [params.embedding_field]},
},
)
def transform_search_vector_store_request(
self,
vector_store_id: str,
query: str | Sequence[str],
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
api_base: str,
litellm_logging_obj: "LiteLLMLoggingObj",
litellm_params: Mapping[str, object],
extra_body: Mapping[str, object] | None = None,
embedding_executor: VectorStoreEmbeddingExecutor | None = None,
) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body
params, k = self._params(litellm_params, vector_store_search_optional_params, extra_body)
embedding: Final = self.query_embedding_executor(embedding_executor, None).embed(
params.require_embedding_model(), self.query_text(query), self.query_embedding_configuration(litellm_params)
)
return self._search_request(vector_store_id, embedding, k, params, api_base)
async def atransform_search_vector_store_request(
self,
vector_store_id: str,
query: str | Sequence[str],
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
api_base: str,
litellm_logging_obj: "LiteLLMLoggingObj",
litellm_params: Mapping[str, object],
extra_body: Mapping[str, object] | None = None,
embedding_executor: VectorStoreEmbeddingExecutor | None = None,
) -> tuple[str, dict[str, object]]: # mutable-ok: the provider contract returns a writable JSON request body
params, k = self._params(litellm_params, vector_store_search_optional_params, extra_body)
embedding: Final = await self.query_embedding_executor(embedding_executor, None).aembed(
params.require_embedding_model(), self.query_text(query), self.query_embedding_configuration(litellm_params)
)
return self._search_request(vector_store_id, embedding, k, params, api_base)
def sign_request(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
request_data: Mapping[str, object],
api_base: str,
api_key: str | None = None,
) -> tuple[dict[str, str], bytes]: # mutable-ok: the shared HTTP handler requires writable headers
region: Final = self._signing_region(optional_params, api_base)
body: Final = json.dumps(request_data)
signed: Final = sign_aws_json_post(
get_credentials=partial(self._signing_credentials, optional_params, region),
service_name=aws_signing_target(api_base)[0],
aws_region_name=region,
url=api_base,
body=body,
headers={
"Content-Type": "application/json",
"x-amz-content-sha256": hashlib.sha256(body.encode()).hexdigest(),
},
)
signed_headers: Final = dict(signed.headers.items())
signed_names: Final = frozenset(name.lower() for name in signed_headers)
return {
**{name: value for name, value in headers.items() if name.lower() not in signed_names},
**signed_headers,
}, body.encode()
def transform_search_vector_store_response(
self, response: httpx.Response, litellm_logging_obj: "LiteLLMLoggingObj"
) -> VectorStoreSearchResponse:
if not response.is_success:
raise BaseLLMException(status_code=response.status_code, message=response.text, headers=response.headers)
try:
hits: Final = _SearchResponse.model_validate_json(response.content).hits.hits
except ValidationError:
raise BaseLLMException(
status_code=502,
message="OpenSearch answered the search without a hits.hits list.",
headers=response.headers,
) from None
logged: Final = _LoggedSearch.model_validate(litellm_logging_obj.model_call_details)
return VectorStoreSearchResponse(
object="vector_store.search_results.page",
search_query=logged.search_query,
data=[_search_result(hit, logged.litellm_params) for hit in hits],
)
def validate_create_vector_store(self) -> NoReturn:
raise _bad_request(_SEARCH_ONLY_MESSAGE)
def transform_create_vector_store_request(
self, vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, api_base: str
) -> NoReturn:
raise _bad_request(_SEARCH_ONLY_MESSAGE)
def transform_create_vector_store_response(self, response: httpx.Response) -> NoReturn:
raise _bad_request(_SEARCH_ONLY_MESSAGE)

3
litellm/llms/sap/chat/transformation.py
View file

@ -11,6 +11,7 @@ import httpx
from pydantic import BaseModel
import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import message_with_file_parts_as_image_urls
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
@ -104,7 +105,7 @@ def _message_role_mapping(role: str) -> type[ChatMessage]:
def _messages_to_sap_template(messages: list[AllMessageValues]) -> list:
template: Final = []
for message in messages:
folded_message = _fold_message_cache_control(message)
folded_message = _fold_message_cache_control(message_with_file_parts_as_image_urls(message))
message_role_class = _message_role_mapping(str(folded_message["role"]))
validated_message = validate_dict(folded_message, message_role_class)
template.append(validated_message)

45
litellm/llms/snowflake/chat/transformation.py
View file

@ -112,7 +112,7 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object:
pdf/text data URIs become document blocks rather than malformed image blocks.
"""
image_url: Final = block.get("image_url")
url: Final = image_url if isinstance(image_url, str) else _image_url_field(image_url, "url")
url: Final = image_url if isinstance(image_url, str) else _str_field(image_url, "url")
if not url:
return block
@ -121,18 +121,31 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object:
if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document"
else create_anthropic_image_param(
image_url if isinstance(image_url, dict) else url,
format=_image_url_field(image_url, "format"),
format=_str_field(image_url, "format"),
is_bedrock_invoke=True,
)
)
return _with_cache_control(converted, block)
def _convert_file_to_anthropic(block: Mapping[str, object]) -> object:
file_data: Final = _str_field(block.get("file"), "file_data")
if not file_data:
return block
return _with_cache_control(
anthropic_process_openai_file_message({"type": "file", "file": {"file_data": file_data}}), block
)
def _with_cache_control(converted: Mapping[str, object], block: Mapping[str, object]) -> Mapping[str, object]:
cache_control: Final = block.get("cache_control")
if cache_control is None:
return converted
return {**converted, "cache_control": cache_control}
def _image_url_field(image_url: object, key: str) -> str | None:
value: Final = image_url.get(key) if isinstance(image_url, dict) else None
def _str_field(container: object, key: str) -> str | None:
value: Final = container.get(key) if isinstance(container, dict) else None
return value if isinstance(value, str) else None
@ -141,15 +154,23 @@ def _data_uri_media_type(url: str) -> str:
return match.group(1) if match else ""
def _convert_image_url_blocks_to_anthropic(content: object) -> object:
def _convert_media_block_to_anthropic(block: object) -> object:
mapping: Final = as_str_mapping(block)
if mapping is None:
return block
match mapping.get("type"):
case "image_url":
return _convert_image_url_to_anthropic(mapping)
case "file":
return _convert_file_to_anthropic(mapping)
case _:
return block
def _convert_media_blocks_to_anthropic(content: object) -> object:
if not isinstance(content, list):
return content
return [
_convert_image_url_to_anthropic(block)
if isinstance(block, Mapping) and block.get("type") == "image_url"
else block
for block in content
]
return [_convert_media_block_to_anthropic(block) for block in content]
def _convert_tool_result_to_anthropic(
@ -391,7 +412,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
conversation.append(
{
"role": role,
"content": _convert_image_url_blocks_to_anthropic(content),
"content": _convert_media_blocks_to_anthropic(content),
}
)

8
litellm/llms/soniox/common_utils.py
View file

@ -3,7 +3,7 @@ Shared utilities for the Soniox provider (https://soniox.com).
"""
from collections.abc import Mapping, Sequence
from typing import Final, TypeAlias
from typing import Final, Literal, TypeAlias
from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
SubtitleToken,
@ -17,6 +17,8 @@ SonioxToken: TypeAlias = Mapping[str, object]
# Soniox API base URL.
SONIOX_API_BASE: Final[str] = "https://api.soniox.com"
SONIOX_REALTIME_MODEL_PREFIX: Final = "stt-rt-"
# Default polling interval in seconds when waiting for an async transcription
# to finish. Mirrors the Soniox SDK default.
SONIOX_DEFAULT_POLL_INTERVAL: Final[float] = 1.0
@ -54,6 +56,10 @@ class SonioxException(BaseLLMException):
"""Provider-specific exception class for Soniox."""
def soniox_health_check_mode(model: str) -> Literal["realtime"] | None:
return "realtime" if model.startswith(SONIOX_REALTIME_MODEL_PREFIX) else None
def get_soniox_api_key(api_key: str | None = None) -> str | None:
"""Resolve the Soniox API key from arg or env var."""
# Local import to avoid a circular import: litellm.secret_managers.main

0
litellm/llms/soniox/realtime/__init__.py Normal file
View file

609
litellm/llms/soniox/realtime/transformation.py Normal file
View file

@ -0,0 +1,609 @@
import asyncio
import time
from collections import deque
from collections.abc import Awaitable, Callable, Mapping
from itertools import chain, groupby
from types import MappingProxyType, TracebackType
from typing import Final
from urllib.parse import urlparse, urlunparse
from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError
from typing_extensions import NotRequired, ReadOnly, Required, Self, TypedDict
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.realtime.transcription_protocol import (
SESSION_UPDATE_EVENT_TYPES,
RealtimeTranscriptionProtocolError,
TranscriptionSessionUpdate,
completed_event,
decode_pcm16_append,
delta_event,
duration_usage,
error_event,
json_object,
parse_transcription_session_update,
speech_event,
transcription_session,
transcription_session_created_event,
uses_server_vad,
)
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
from litellm.llms.custom_httpx.http_handler import realtime_ssl_for_url
from litellm.llms.soniox.common_utils import SonioxException, get_soniox_api_key
from litellm.types.llms.openai import (
OpenAIRealtimeErrorEvent,
OpenAIRealtimeEvents,
OpenAIRealtimeInputAudioTranscriptionCompleted,
OpenAIRealtimeInputAudioTranscriptionDelta,
OpenAIRealtimeTranscriptionSessionCreated,
)
from litellm.types.realtime import (
RealtimeInputAudioTranscriptionUsage,
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
)
DEFAULT_SONIOX_REALTIME_URL: Final = "wss://stt-rt.soniox.com/transcribe-websocket"
WEBSOCKET_SCHEMES: Final = MappingProxyType({"wss": "wss", "ws": "ws", "https": "wss", "http": "ws"})
DEFAULT_SAMPLE_RATE: Final = 24_000
KEEPALIVE_INTERVAL_SECONDS: Final = 5.0
_MAX_PENDING_FRAMES: Final = 64
_FINALIZE: Final = '{"type":"finalize"}'
_KEEPALIVE: Final = '{"type":"keepalive"}'
_END_STREAM: Final = ""
_HEALTH_CHECK_SILENCE: Final = bytes(DEFAULT_SAMPLE_RATE // 5)
_CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM))
SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}'
_ENDPOINT_TOKEN: Final = "<end>"
_FINALIZED_TOKEN: Final = "<fin>"
class SonioxProtocolError(RealtimeTranscriptionProtocolError):
pass
class SonioxRealtimeOptions(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore")
language_hints: tuple[str, ...] | None = None
language_hints_strict: bool | None = None
enable_language_identification: bool | None = None
enable_speaker_diarization: bool | None = None
context: JsonValue | None = None
translation: JsonValue | None = None
client_reference_id: str | None = None
max_endpoint_delay_ms: int | None = None
endpoint_sensitivity: float | None = None
endpoint_latency_adjustment_level: int | None = None
class SonioxStartRequest(SonioxRealtimeOptions):
model: str
audio_format: str
sample_rate: int
num_channels: int
enable_endpoint_detection: bool
class SonioxSpeechLabels(TypedDict, total=False):
speaker: ReadOnly[str]
language: ReadOnly[str]
class SonioxTranscriptSegment(SonioxSpeechLabels):
text: Required[ReadOnly[str]]
class SonioxTranscriptionDelta(OpenAIRealtimeInputAudioTranscriptionDelta, SonioxSpeechLabels):
pass
class SonioxTranscriptionCompleted(OpenAIRealtimeInputAudioTranscriptionCompleted):
segments: NotRequired[ReadOnly[tuple[SonioxTranscriptSegment, ...]]]
class _SonioxToken(BaseModel):
model_config = ConfigDict(frozen=True)
text: str
is_final: bool = False
translation_status: str | None = None
speaker: str | None = None
language: str | None = None
@property
def labelled(self) -> bool:
return self.speaker is not None or self.language is not None
class _SonioxResponse(BaseModel):
model_config = ConfigDict(frozen=True)
tokens: tuple[_SonioxToken, ...] = ()
total_audio_proc_ms: int | None = None
error_code: int | None = None
error_message: str | None = None
finished: bool = False
def _sample_rate(update: TranscriptionSessionUpdate | None) -> int:
audio_format: Final = None if update is None else update.audio_format
if audio_format is None:
return DEFAULT_SAMPLE_RATE
if not audio_format.is_pcm16:
raise SonioxProtocolError("Soniox realtime requires pcm16 input audio")
if audio_format.channels not in (None, 1):
raise SonioxProtocolError("Soniox realtime requires mono input audio")
if audio_format.rate is None:
return DEFAULT_SAMPLE_RATE
if audio_format.rate <= 0:
raise SonioxProtocolError("sample rate must be positive")
return audio_format.rate
def _language_hints(
language: str | None, configured: tuple[str, ...] | None, strict: bool | None
) -> tuple[str, ...] | None:
if not language or (strict and configured):
return configured
return (language, *(hint for hint in configured or () if hint != language))
def _uses_endpoint_detection(update: TranscriptionSessionUpdate, drop_params: bool) -> bool:
return uses_server_vad(update, "Soniox realtime", drop_params, SonioxProtocolError)
def _ignored_settings(update: TranscriptionSessionUpdate) -> tuple[str, ...]:
vad_tuning: Final = (f"turn_detection.{key}" for key in sorted(update.turn_detection or {}) if key != "type")
return (*update.unsupported_transcription_keys, *vad_tuning)
def build_start_request(
model: str, update: TranscriptionSessionUpdate | None, options: SonioxRealtimeOptions, drop_params: bool = False
) -> SonioxStartRequest:
return SonioxStartRequest(
**options.model_dump(exclude={"language_hints"}),
language_hints=_language_hints(
None if update is None else update.language, options.language_hints, options.language_hints_strict
),
model=model,
audio_format="pcm_s16le",
sample_rate=_sample_rate(update),
num_channels=1,
enable_endpoint_detection=update is None or _uses_endpoint_detection(update, drop_params),
)
def _session_created(session_id: str, start_request: SonioxStartRequest) -> OpenAIRealtimeTranscriptionSessionCreated:
session: Final = transcription_session(
session_id=session_id,
model=start_request.model,
sample_rate=start_request.sample_rate,
language=start_request.language_hints[0] if start_request.language_hints else None,
server_vad=start_request.enable_endpoint_detection,
)
return transcription_session_created_event(session)
def _changes_started_rate(update: TranscriptionSessionUpdate, started: SonioxStartRequest) -> bool:
audio_format: Final = update.audio_format
if audio_format is None:
return False
requested_rate: Final = _sample_rate(update)
return audio_format.rate is not None and requested_rate != started.sample_rate
def _changes_started_stream(update: TranscriptionSessionUpdate, started: SonioxStartRequest, drop_params: bool) -> bool:
turn_detection_set: Final = update.turn_detection is not None or update.turn_detection_disabled
return (
_changes_started_rate(update, started)
or (
update.language is not None
and _language_hints(update.language, started.language_hints, started.language_hints_strict)
!= started.language_hints
)
or (turn_detection_set and _uses_endpoint_detection(update, drop_params) != started.enable_endpoint_detection)
)
def build_soniox_realtime_url(api_base: str | None) -> str:
if api_base is None:
return DEFAULT_SONIOX_REALTIME_URL
parsed: Final = urlparse(api_base)
scheme: Final = WEBSOCKET_SCHEMES.get(parsed.scheme)
if scheme is None:
raise ValueError("Soniox realtime api_base must be a wss://, ws://, https://, or http:// URL")
return urlunparse(parsed._replace(scheme=scheme))
def _labels(token: _SonioxToken) -> SonioxSpeechLabels:
speaker: Final[SonioxSpeechLabels] = {} if token.speaker is None else {"speaker": token.speaker}
language: Final[SonioxSpeechLabels] = {} if token.language is None else {"language": token.language}
return {**speaker, **language}
def _delta(item_id: str, token: _SonioxToken) -> SonioxTranscriptionDelta:
return {**delta_event(item_id, token.text), **_labels(token)}
def _segment(run: tuple[_SonioxToken, ...]) -> SonioxTranscriptSegment:
return {**_labels(run[0]), "text": "".join(token.text for token in run).strip()}
def _segments(tokens: tuple[_SonioxToken, ...]) -> tuple[SonioxTranscriptSegment, ...]:
return tuple(_segment(tuple(run)) for _, run in groupby(tokens, key=lambda token: (token.speaker, token.language)))
def _new_item_id() -> str:
return f"item_{uuid.uuid4().hex}"
def _completed(
item_id: str, tokens: tuple[_SonioxToken, ...], usage: RealtimeInputAudioTranscriptionUsage | None
) -> OpenAIRealtimeInputAudioTranscriptionCompleted:
completed: Final = completed_event(item_id, "".join(token.text for token in tokens).strip(), usage)
if not any(token.labelled for token in tokens):
return completed
labelled: Final[SonioxTranscriptionCompleted] = {**completed, "segments": _segments(tokens)}
return labelled
def _soniox_error(response: _SonioxResponse) -> SonioxException | None:
if response.error_code is None and response.error_message is None:
return None
return SonioxException(
status_code=response.error_code or 500,
message=f"Soniox realtime error {response.error_code}: {response.error_message}",
)
class SonioxEventTransformer:
def __init__(self, *, drops_originals: bool) -> None:
self._drops_originals: Final = drops_originals
self._item_id: str | None = None
self._final_tokens: Final[list[_SonioxToken]] = [] # mutable-ok: a turn's tokens append in O(1) each
self._finalizes: Final[deque[bool]] = deque() # mutable-ok: pending finalizes queue and pop in O(1) each
self._oldest_commit_answered: bool = False
self._processed_ms: int = 0
self._billed_ms: int = 0
self._refusal: SonioxException | None = None
@property
def refusal(self) -> SonioxException | None:
return self._refusal
def transform(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]:
try:
response: Final = _SonioxResponse.model_validate_json(payload)
except ValidationError:
raise SonioxProtocolError("Soniox returned an invalid realtime response") from None
self._bill(response.total_audio_proc_ms)
error: Final = _soniox_error(response)
if error is not None:
return (self._report(error),)
return tuple(chain.from_iterable(self._token_events(token) for token in response.tokens))
def queue_finalize(self, *, discards_turn: bool) -> None:
if discards_turn and not self._finalizes:
self._item_id = None
self._final_tokens.clear()
self._finalizes.append(discards_turn)
def take_unbilled_usage(self, stream_ms: int = 0) -> RealtimeInputAudioTranscriptionUsage | None:
billable_ms: Final = max(self._processed_ms, stream_ms)
unbilled_ms: Final = billable_ms - self._billed_ms
if unbilled_ms <= 0:
return None
self._billed_ms = billable_ms
return duration_usage(unbilled_ms / 1000)
def _report(self, error: SonioxException) -> OpenAIRealtimeErrorEvent:
if self._processed_ms == 0:
self._refusal = error
return error_event(error.message)
def _bill(self, total_audio_proc_ms: int | None) -> None:
if total_audio_proc_ms is not None and total_audio_proc_ms > self._processed_ms:
self._processed_ms = total_audio_proc_ms
def _token_events(self, token: _SonioxToken) -> tuple[OpenAIRealtimeEvents, ...]:
if token.text == _FINALIZED_TOKEN:
return self._finalized()
if self._discarding:
return ()
if token.text == _ENDPOINT_TOKEN:
return self._end_turn()
started: Final = self._start()
if not token.is_final or (self._drops_originals and token.translation_status == "original"):
return started
self._final_tokens.append(token)
return (*started, _delta(self._current_item_id(), token))
@property
def _discarding(self) -> bool:
return bool(self._finalizes) and self._finalizes[0]
def _finalized(self) -> tuple[OpenAIRealtimeEvents, ...]:
discarded: Final = self._finalizes.popleft() if self._finalizes else None
answered: Final = self._oldest_commit_answered
self._oldest_commit_answered = False
if discarded:
return ()
if self._item_id is not None:
return self._complete(speech_stopped=False)
if discarded is None or answered:
return ()
return (_completed(_new_item_id(), (), self.take_unbilled_usage()),)
def _end_turn(self) -> tuple[OpenAIRealtimeEvents, ...]:
events: Final = self._complete(speech_stopped=True)
if events and self._finalizes:
self._oldest_commit_answered = True
return events
def _start(self) -> tuple[OpenAIRealtimeEvents, ...]:
if self._item_id is not None:
return ()
item_id: Final = _new_item_id()
self._item_id = item_id
return (speech_event("input_audio_buffer.speech_started", item_id),)
def _current_item_id(self) -> str:
assert self._item_id is not None
return self._item_id
def _complete(self, *, speech_stopped: bool) -> tuple[OpenAIRealtimeEvents, ...]:
item_id: Final = self._item_id
if item_id is None:
return ()
tokens: Final = tuple(self._final_tokens)
self._item_id = None
self._final_tokens.clear()
stopped: Final[tuple[OpenAIRealtimeEvents, ...]] = (
(speech_event("input_audio_buffer.speech_stopped", item_id),) if speech_stopped else ()
)
return (*stopped, _completed(item_id, tokens, self.take_unbilled_usage()))
class SonioxRealtimeBackend:
def __init__(
self,
inner: RealtimeBackend,
*,
keepalive_interval: float = KEEPALIVE_INTERVAL_SECONDS,
clock: Callable[[], float] = time.monotonic,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
) -> None:
self._inner: Final = inner
self._interval: Final = keepalive_interval
self._clock: Final = clock
self._sleep: Final = sleep
self._frames: Final[asyncio.Queue[str | bytes | Exception]] = asyncio.Queue(maxsize=_MAX_PENDING_FRAMES)
self._last_sent: float = clock()
self._started: bool = False
self._ended: bool = False
self._tasks: tuple[asyncio.Task[None], ...] = ()
async def __aenter__(self) -> Self:
await self._inner.__aenter__()
self._tasks = (asyncio.create_task(self._pump()), asyncio.create_task(self._keepalive()))
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
self._stop()
await self._inner.__aexit__(exc_type, exc_value, traceback)
async def send(self, message: str | bytes) -> None:
self._last_sent = self._clock()
self._ended = self._ended or message == _END_STREAM
await self._inner.send(message)
if not self._started and isinstance(message, str) and message not in _CONTROL_FRAMES:
self._started = True
await self._frames.put(SESSION_STARTED_FRAME)
async def recv(self, decode: bool | None = None) -> str | bytes:
frame: Final = await self._frames.get()
if isinstance(frame, Exception):
raise frame
return frame
async def close(self) -> None:
self._stop()
await self._inner.close()
def _stop(self) -> None:
for task in self._tasks:
task.cancel()
async def _pump(self) -> None:
while True:
try:
frame = await self._inner.recv()
except Exception as e: # noqa: BLE001 # recv() re-raises it, so an upstream failure never hangs the reader
await self._frames.put(e)
return
await self._frames.put(frame)
async def _keepalive(self) -> None:
while not self._ended:
await self._sleep(self._interval)
if self._ended or self._clock() - self._last_sent < self._interval:
continue
try:
await self.send(_KEEPALIVE)
except Exception as e: # noqa: BLE001 # stops only keepalives; the pump reports the broken socket
verbose_logger.debug("Soniox realtime: stopping keepalive after send failed: %s", e)
return
class SonioxRealtimeConfig(BaseRealtimeConfig):
def __init__(
self,
*,
options: SonioxRealtimeOptions | None = None,
wrap: Callable[[RealtimeBackend], RealtimeBackend] = SonioxRealtimeBackend,
clock: Callable[[], float] = time.monotonic,
drop_params: bool = False,
) -> None:
self._options: Final = options or SonioxRealtimeOptions()
self._drop_params: Final = drop_params
self._wrap: Final = wrap
self._clock: Final = clock
self._transformer: Final = SonioxEventTransformer(drops_originals=self._options.translation is not None)
self._start_request: SonioxStartRequest | None = None
self._session_id: str = f"sess_{uuid.uuid4().hex}"
self._started_at: float | None = None
@classmethod
def from_litellm_params(cls, litellm_params: Mapping[str, object], *, drop_params: bool = False) -> Self:
return cls(options=SonioxRealtimeOptions.model_validate(dict(litellm_params)), drop_params=drop_params)
def requires_session_configuration(self) -> bool:
return True
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: BaseRealtimeConfig contract
model: str,
api_key: str | None = None,
) -> dict[str, str]: # mutable-ok: BaseRealtimeConfig contract
resolved_key: Final = get_soniox_api_key(api_key)
if not resolved_key:
raise ValueError("Missing Soniox API key. Set SONIOX_API_KEY or api_key in the deployment litellm_params")
return {**headers, "Authorization": f"Bearer {resolved_key}"}
def get_complete_url(self, api_base: str | None, model: str, api_key: str | None = None) -> str:
return build_soniox_realtime_url(api_base)
def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend:
return self._wrap(backend)
def transform_session_created_event(
self,
model: str,
logging_session_id: str,
session_configuration_request: str | None = None,
) -> OpenAIRealtimeTranscriptionSessionCreated:
self._session_id = logging_session_id
return _session_created(logging_session_id, build_start_request(model, None, self._options))
def transform_realtime_request(
self,
message: str,
model: str,
session_configuration_request: str | None = None,
) -> tuple[str | bytes, ...]:
request: Final = json_object(message, SonioxProtocolError)
event_type: Final = request.get("type")
if event_type is None:
raise SonioxProtocolError("Soniox realtime takes OpenAI realtime events, and this message has no type")
if event_type in SESSION_UPDATE_EVENT_TYPES:
update: Final = parse_transcription_session_update(message, SonioxProtocolError)
ignored: Final = _ignored_settings(update)
if ignored:
verbose_logger.warning("Soniox realtime: ignoring unsupported session settings %s", ignored)
return self._start(model, update)
if event_type == "input_audio_buffer.append":
audio: Final = decode_pcm16_append(request.get("audio"), error=SonioxProtocolError)
return (*self._start(model, None), audio)
if event_type == "input_audio_buffer.commit":
return self._finalize(model, discards_turn=False)
if event_type == "input_audio_buffer.clear":
return () if self._start_request is None else self._finalize(model, discards_turn=True)
if event_type == "input_audio_buffer.end":
return (*self._start(model, None), _END_STREAM)
verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type)
return ()
def _finalize(self, model: str, *, discards_turn: bool) -> tuple[str, ...]:
start: Final = self._start(model, None)
self._transformer.queue_finalize(discards_turn=discards_turn)
return (*start, _FINALIZE)
def session_refusal(self) -> Exception | None:
return self._transformer.refusal
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
started_at: Final = self._started_at
if self._transformer.refusal is not None:
return None
if started_at is None:
return self._transformer.take_unbilled_usage()
return self._transformer.take_unbilled_usage(int((self._clock() - started_at) * 1000))
def transform_realtime_response(
self,
message: str | bytes,
model: str,
logging_obj: LiteLLMLoggingObj,
realtime_response_transform_input: RealtimeResponseTransformInput,
) -> RealtimeResponseTypedDict:
payload: Final = message.decode("utf-8") if isinstance(message, bytes) else message
result: Final[RealtimeResponseTypedDict] = {
"response": list(self._backend_events(payload)),
"current_output_item_id": realtime_response_transform_input.get("current_output_item_id"),
"current_response_id": realtime_response_transform_input.get("current_response_id"),
"current_delta_chunks": realtime_response_transform_input.get("current_delta_chunks"),
"current_conversation_id": realtime_response_transform_input.get("current_conversation_id"),
"current_item_chunks": realtime_response_transform_input.get("current_item_chunks"),
"current_delta_type": realtime_response_transform_input.get("current_delta_type"),
"session_configuration_request": realtime_response_transform_input.get("session_configuration_request"),
}
return result
def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]:
if payload != SESSION_STARTED_FRAME:
return self._transformer.transform(payload)
assert self._start_request is not None
return (_session_created(self._session_id, self._start_request),)
def _start(self, model: str, update: TranscriptionSessionUpdate | None) -> tuple[str, ...]:
started: Final = self._start_request
if started is not None:
if update is not None and _changes_started_stream(update, started, self._drop_params):
raise SonioxProtocolError("Soniox realtime can't change the session once the stream has started")
return ()
start_request: Final = build_start_request(model, update, self._options, self._drop_params)
self._start_request = start_request
self._started_at = self._clock()
return (start_request.model_dump_json(exclude_none=True),)
def _finishes_stream(frame: str | bytes) -> bool:
response: Final = _SonioxResponse.model_validate_json(frame)
error: Final = _soniox_error(response)
if error is not None:
raise error
return response.finished
async def soniox_realtime_health_check(
model: str, api_base: str | None, api_key: str | None, options: SonioxRealtimeOptions
) -> bool:
import websockets
config: Final = SonioxRealtimeConfig()
url: Final = config.get_complete_url(api_base, model)
headers: Final = config.validate_environment({}, model, api_key)
start_request: Final = build_start_request(model, None, options)
async with websockets.connect(
url,
additional_headers=headers,
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=realtime_ssl_for_url(url),
) as backend:
await backend.send(start_request.model_dump_json(exclude_none=True))
await backend.send(_HEALTH_CHECK_SILENCE)
await backend.send(_END_STREAM)
async for frame in backend:
if _finishes_stream(frame):
return True
raise SonioxException(status_code=502, message="Soniox closed the realtime stream without finishing it")

3
litellm/llms/vertex_ai/vertex_ai_partner_models/main.py
View file

@ -9,6 +9,7 @@ import httpx
import litellm
from litellm import LlmProviders
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.types.llms.vertex_ai import VertexPartnerProvider
from litellm.utils import ModelResponse
@ -217,7 +218,7 @@ class VertexAIPartnerModels(VertexBase):
return base_llm_http_handler.completion(
model=model,
stream=stream,
messages=messages,
messages=list(messages_with_file_parts_as_image_urls(messages)),
acompletion=acompletion,
api_base=api_base,
model_response=model_response,

163
litellm/main.py
View file

@ -108,6 +108,10 @@ from litellm.litellm_core_utils.mock_functions import (
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
messages_have_pdf_file_part,
messages_without_pdf_file_parts,
)
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
@ -191,11 +195,13 @@ from litellm.utils import (
get_requester_metadata,
get_secret,
get_standard_openai_params,
is_explicitly_disabled_factory,
mock_completion_streaming_obj,
pre_process_non_default_params,
read_config_args,
should_run_mock_completion,
supports_httpx_timeout,
supports_pdf_input,
validate_and_fix_openai_messages,
validate_and_fix_openai_tools,
validate_and_fix_thinking_param,
@ -393,6 +399,125 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
return assert_never(control)
PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
def _cost_map_pdf_input_support(model: str, custom_llm_provider: str) -> bool | None:
if custom_llm_provider in PROVIDERS_WITHOUT_DOCUMENT_INPUT or is_explicitly_disabled_factory(
model, custom_llm_provider, "supports_pdf_input"
):
return False
if supports_pdf_input(model, custom_llm_provider):
return True
return None
def _deployment_pdf_input_support(model: str, base_model: str, custom_llm_provider: str) -> bool | None:
supports: Final = (
_cost_map_pdf_input_support(model, custom_llm_provider),
_cost_map_pdf_input_support(base_model, custom_llm_provider),
)
if False in supports:
return False
if True in supports:
return True
return None
def _declared_pdf_input_support(
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None, base_model: str | None
) -> bool | None:
match model_info:
case {"supports_pdf_input": bool() as declared}:
return declared
case _ if base_model:
return _deployment_pdf_input_support(model, base_model, custom_llm_provider)
case _:
return _cost_map_pdf_input_support(model, custom_llm_provider)
def _pdf_input_unreadable(
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object] | None,
base_model: str | None,
drop_params: bool,
) -> bool:
match _declared_pdf_input_support(model, custom_llm_provider, model_info, base_model):
case False:
return True
case None:
return drop_params and custom_llm_provider in PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT
case True:
return False
def _messages_without_unreadable_pdfs(
messages: Sequence[AllMessageValues],
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object] | None,
base_model: str | None,
drop_params: bool,
) -> tuple[AllMessageValues, ...] | None:
if not messages_have_pdf_file_part(messages) or not _pdf_input_unreadable(
model, custom_llm_provider, model_info, base_model, drop_params
):
return None
if not drop_params:
raise litellm.BadRequestError(
message=(
f"{custom_llm_provider}/{model} does not accept PDF input. "
"Remove the PDF, or set drop_params=True to send the request without it"
),
model=model,
llm_provider=custom_llm_provider,
)
readable_messages: Final = messages_without_pdf_file_parts(messages)
if not any(message["role"] == "user" for message in readable_messages):
raise litellm.BadRequestError(
message=(
f"{custom_llm_provider}/{model} does not accept PDF input, "
"and no user message is left once the PDF is dropped"
),
model=model,
llm_provider=custom_llm_provider,
)
return readable_messages
def _messages_readable_by_model(
messages: Sequence[AllMessageValues],
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object] | None,
base_model: str | None,
drop_params: bool,
ensure_alternating_roles: bool | None,
user_continue_message: ChatCompletionUserMessage | None,
assistant_continue_message: ChatCompletionAssistantMessage | None,
) -> tuple[AllMessageValues, ...] | None:
pdf_free_messages: Final = _messages_without_unreadable_pdfs(
messages=messages,
model=model,
custom_llm_provider=custom_llm_provider,
model_info=model_info,
base_model=base_model,
drop_params=drop_params,
)
if pdf_free_messages is None:
return None
return tuple(
get_completion_messages(
messages=list(pdf_free_messages),
ensure_alternating_roles=ensure_alternating_roles or False,
user_continue_message=user_continue_message,
assistant_continue_message=assistant_continue_message,
)
)
####### COMPLETION ENDPOINTS ################
@ -4234,7 +4359,6 @@ def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchRe
_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
@ -5558,9 +5682,15 @@ def completion(
)
headers = kwargs.get("headers", None) or extra_headers
ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None)
user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None)
assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None)
ensure_alternating_roles: Final = cast( # cast-ok: read from untyped kwargs
bool | None, kwargs.get("ensure_alternating_roles", None)
)
user_continue_message: Final = cast( # cast-ok: read from untyped kwargs
ChatCompletionUserMessage | None, kwargs.get("user_continue_message", None)
)
assistant_continue_message: Final = cast( # cast-ok: read from untyped kwargs
ChatCompletionAssistantMessage | None, kwargs.get("assistant_continue_message", None)
)
headers = {} if headers is None else dict(headers)
if extra_headers is not None:
headers.update(extra_headers)
@ -5928,15 +6058,17 @@ def completion(
def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]:
return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params)
optional_params = optional_params_for_model_info(
validated_model_info: Final = (
_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None
)
deployment_base_model: Final = base_model if isinstance(base_model, str) else None
optional_params = optional_params_for_model_info(validated_model_info)
fakes_stream: Final = partial(
_completion_fakes_stream,
model,
custom_llm_provider,
provider_config,
base_model=base_model if isinstance(base_model, str) else None,
base_model=deployment_base_model,
bridges_to_responses_api=bridges_to_responses_api is True,
)
resolve_timeout: Final = partial(
@ -5974,16 +6106,29 @@ def completion(
functions_unsupported_model: Final[list[object]] = TypeAdapter(list[object]).validate_python(
optional_params.pop("functions_unsupported_model")
)
request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
messages = function_call_prompt(
messages=messages,
functions=functions_unsupported_model,
model=model,
llm_provider=custom_llm_provider,
drop_params=litellm.drop_params is True
or normalize_drop_params(request_kwargs.get("drop_params")) is True,
drop_params=drop_unsupported_reasoning,
)
readable_messages: Final = _messages_readable_by_model(
messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List
model=model,
custom_llm_provider=custom_llm_provider,
model_info=validated_model_info,
base_model=deployment_base_model,
drop_params=drop_unsupported_reasoning,
ensure_alternating_roles=ensure_alternating_roles,
user_continue_message=user_continue_message,
assistant_continue_message=assistant_continue_message,
)
messages = ( # rebind-ok: completion() rewrites messages step by step
messages if readable_messages is None else list(readable_messages)
)
# For logging - save the values of the litellm-specific params passed in
requested_litellm_params: Final = get_litellm_params(
acompletion=acompletion,

241
litellm/model_prices_and_context_window_backup.json
View file

@ -4049,6 +4049,7 @@
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": false,
"supports_response_schema": true,
"supports_tool_choice": true
},
@ -12437,7 +12438,8 @@
"supported_output_modalities": [
"text"
],
"supports_native_streaming": true
"supports_native_streaming": true,
"supports_pdf_input": false
},
"azure_ai/FLUX-1.1-pro": {
"deprecation_date": "2027-01-05",
@ -12502,6 +12504,7 @@
"output_cost_per_token": 1.85e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12516,6 +12519,7 @@
"output_cost_per_token": 3.828e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12531,6 +12535,7 @@
"output_cost_per_token": 3.52e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12546,6 +12551,7 @@
"output_cost_per_token": 4.84e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12560,6 +12566,7 @@
"output_cost_per_token": 4.84e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12574,6 +12581,7 @@
"output_cost_per_token": 7.26e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12594,6 +12602,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12616,6 +12625,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -12638,6 +12648,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -12660,6 +12671,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -12687,6 +12699,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -12703,6 +12716,7 @@
"output_cost_per_token": 1.32e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12724,6 +12738,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -12744,6 +12759,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -12765,6 +12781,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12833,6 +12850,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -12847,6 +12865,7 @@
"output_cost_per_token": 7.1e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": {
@ -12859,6 +12878,7 @@
"output_cost_per_token": 1e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": true
},
@ -12872,6 +12892,7 @@
"output_cost_per_token": 7.8e-07,
"source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": true
},
@ -12884,6 +12905,7 @@
"mode": "chat",
"output_cost_per_token": 3.7e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/Meta-Llama-3.1-70B-Instruct": {
@ -12895,6 +12917,7 @@
"mode": "chat",
"output_cost_per_token": 3.54e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/Phi-3-medium-128k-instruct": {
@ -12906,6 +12929,7 @@
"mode": "chat",
"output_cost_per_token": 6.8e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -12918,6 +12942,7 @@
"mode": "chat",
"output_cost_per_token": 6.8e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -12930,6 +12955,7 @@
"mode": "chat",
"output_cost_per_token": 5.2e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -12942,6 +12968,7 @@
"mode": "chat",
"output_cost_per_token": 5.2e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -12954,6 +12981,7 @@
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -12966,6 +12994,7 @@
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -12978,6 +13007,7 @@
"mode": "chat",
"output_cost_per_token": 6.4e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -12990,6 +13020,7 @@
"mode": "chat",
"output_cost_per_token": 5.2e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -13002,6 +13033,7 @@
"mode": "chat",
"output_cost_per_token": 5.2e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": true
},
@ -13015,6 +13047,7 @@
"output_cost_per_token": 5e-07,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": false
},
@ -13027,7 +13060,8 @@
"mode": "chat",
"output_cost_per_token": 3e-07,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true
"supports_function_calling": true,
"supports_pdf_input": false
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
@ -13041,6 +13075,7 @@
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_audio_input": true,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
@ -13052,7 +13087,8 @@
"mode": "chat",
"output_cost_per_token": 3e-07,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true
"supports_function_calling": true,
"supports_pdf_input": false
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
@ -13064,6 +13100,7 @@
"output_cost_per_token": 5e-07,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_reasoning": true
},
@ -13106,6 +13143,7 @@
"output_cost_per_token": 1e-05,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/doc-intelligence/prebuilt-read": {
@ -13144,6 +13182,7 @@
"mode": "chat",
"output_cost_per_token": 5.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_reasoning": true,
"supports_tool_choice": true
},
@ -13202,6 +13241,7 @@
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -13217,6 +13257,7 @@
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -13230,6 +13271,7 @@
"mode": "chat",
"output_cost_per_token": 4.56e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/deepseek-v4-pro": {
@ -13242,6 +13284,7 @@
"output_cost_per_token": 3.48e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_reasoning": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 1.45e-07,
@ -13257,6 +13300,7 @@
"output_cost_per_token": 5.1e-07,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_reasoning": true,
"supports_tool_choice": true,
"cache_read_input_token_cost": 2.8e-08,
@ -13273,6 +13317,7 @@
"output_cost_per_token": 1.32e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -13306,6 +13351,7 @@
"output_cost_per_token": 1.5e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_web_search": true
@ -13323,6 +13369,7 @@
"output_cost_per_token_above_200k_tokens": 5e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -13343,6 +13390,7 @@
"output_cost_per_token_above_200k_tokens": 1.2e-05,
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -13361,6 +13409,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -13379,6 +13428,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -13395,6 +13445,7 @@
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_web_search": true
@ -13409,6 +13460,7 @@
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
@ -13424,6 +13476,7 @@
"output_cost_per_token": 1.5e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_web_search": true
@ -13436,7 +13489,8 @@
"max_output_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 0.00971,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models"
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false
},
"azure_ai/jamba-instruct": {
"deprecation_date": "2025-03-01",
@ -13447,6 +13501,7 @@
"mode": "chat",
"output_cost_per_token": 7e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/kimi-k2.5": {
@ -13459,6 +13514,7 @@
"output_cost_per_token": 3e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true,
@ -13482,6 +13538,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -13498,6 +13555,7 @@
"output_cost_per_token": 4e-08,
"source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/mistral-large": {
@ -13510,6 +13568,7 @@
"output_cost_per_token": 1.2e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/mistral-large-2407": {
@ -13522,6 +13581,7 @@
"output_cost_per_token": 6e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/mistral-large-latest": {
@ -13533,6 +13593,7 @@
"output_cost_per_token": 6e-06,
"source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/mistral-large-3": {
@ -13544,6 +13605,7 @@
"output_cost_per_token": 1.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": true
},
@ -13557,6 +13619,7 @@
"output_cost_per_token": 2e-06,
"supports_assistant_prefill": true,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/mistral-nemo": {
@ -13568,7 +13631,8 @@
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_function_calling": true
"supports_function_calling": true,
"supports_pdf_input": false
},
"azure_ai/mistral-small": {
"deprecation_date": "2025-07-31",
@ -13580,6 +13644,7 @@
"output_cost_per_token": 3e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"azure_ai/mistral-small-2503": {
@ -13591,6 +13656,7 @@
"mode": "chat",
"output_cost_per_token": 3e-07,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_vision": true
},
@ -52006,6 +52072,7 @@
],
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -52026,6 +52093,7 @@
],
"supports_assistant_prefill": false,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -52777,6 +52845,7 @@
"output_cost_per_token": 2.5e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true,
"supports_web_search": true
},
@ -52794,6 +52863,7 @@
"global"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_reasoning": true,
"supports_tool_choice": true
},
@ -52811,6 +52881,7 @@
"global"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -52989,6 +53060,7 @@
"output_cost_per_token": 3.6e-07,
"output_cost_per_token_batches": 1.8e-07,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_pdf_input": false,
"supports_reasoning": true
},
"vertex_ai/openai/gpt-oss-20b-maas": {
@ -53000,6 +53072,7 @@
"mode": "chat",
"output_cost_per_token": 2.5e-07,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supports_pdf_input": false,
"supports_reasoning": true,
"cache_read_input_token_cost": 7e-09,
"input_cost_per_token_batches": 3.5e-08,
@ -53131,6 +53204,7 @@
"us-south1"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": {
@ -53149,6 +53223,7 @@
"global"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
@ -53164,6 +53239,7 @@
"global"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
@ -53179,6 +53255,7 @@
"global"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_tool_choice": true
},
"vertex_ai/veo-3.1-generate-preview": {
@ -61177,6 +61254,126 @@
],
"supports_audio_input": true
},
"soniox/stt-rt-v5": {
"input_cost_per_second": 3.333e-05,
"litellm_provider": "soniox",
"metadata": {
"calculation": "$0.12/3600 seconds = $0.00003333 per second",
"original_pricing_per_hour": 0.12
},
"mode": "audio_transcription",
"source": "https://soniox.com/pricing",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"soniox/stt-rt-preview": {
"input_cost_per_second": 3.333e-05,
"litellm_provider": "soniox",
"metadata": {
"calculation": "$0.12/3600 seconds = $0.00003333 per second",
"original_pricing_per_hour": 0.12
},
"mode": "audio_transcription",
"source": "https://soniox.com/pricing",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"soniox/stt-rt-preview-v2": {
"input_cost_per_second": 3.333e-05,
"litellm_provider": "soniox",
"metadata": {
"calculation": "$0.12/3600 seconds = $0.00003333 per second",
"original_pricing_per_hour": 0.12
},
"mode": "audio_transcription",
"source": "https://soniox.com/pricing",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"soniox/stt-rt-v3": {
"input_cost_per_second": 3.333e-05,
"litellm_provider": "soniox",
"metadata": {
"calculation": "$0.12/3600 seconds = $0.00003333 per second",
"original_pricing_per_hour": 0.12
},
"mode": "audio_transcription",
"source": "https://soniox.com/pricing",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"soniox/stt-rt-v3-preview": {
"input_cost_per_second": 3.333e-05,
"litellm_provider": "soniox",
"metadata": {
"calculation": "$0.12/3600 seconds = $0.00003333 per second",
"original_pricing_per_hour": 0.12
},
"mode": "audio_transcription",
"source": "https://soniox.com/pricing",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"soniox/stt-rt-v4": {
"input_cost_per_second": 3.333e-05,
"litellm_provider": "soniox",
"metadata": {
"calculation": "$0.12/3600 seconds = $0.00003333 per second",
"original_pricing_per_hour": 0.12
},
"mode": "audio_transcription",
"source": "https://soniox.com/pricing",
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": {
"litellm_provider": "tensormesh",
"mode": "chat",
@ -67202,6 +67399,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -67216,6 +67414,7 @@
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -67258,6 +67457,7 @@
"output_cost_per_token": 1.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -67272,6 +67472,7 @@
"output_cost_per_token": 3.1e-07,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -67286,6 +67487,7 @@
"output_cost_per_token": 5.5e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -67300,6 +67502,7 @@
"output_cost_per_token": 6.25e-07,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -67316,6 +67519,7 @@
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -67331,6 +67535,7 @@
"output_cost_per_token": 3.2e-06,
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_reasoning": true,
"supports_tool_choice": true
},
@ -67349,6 +67554,7 @@
"supported_output_modalities": [
"text"
],
"supports_pdf_input": false,
"supports_response_schema": true,
"supports_vision": true
},
@ -67358,7 +67564,8 @@
"output_cost_per_token": 5.4e-06,
"litellm_provider": "azure_ai",
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_pdf_input": false
},
"azure_ai/deepseek-v3-0324": {
"deprecation_date": "2026-07-13",
@ -67366,7 +67573,8 @@
"output_cost_per_token": 4.56e-06,
"litellm_provider": "azure_ai",
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_pdf_input": false
},
"azure_ai/deepseek-v3.1": {
"deprecation_date": "2026-07-13",
@ -67374,7 +67582,8 @@
"output_cost_per_token": 4.94e-06,
"litellm_provider": "azure_ai",
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_pdf_input": false
},
"azure_ai/grok-3": {
"deprecation_date": "2026-05-01",
@ -67382,7 +67591,8 @@
"output_cost_per_token": 1.5e-05,
"litellm_provider": "azure_ai",
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_pdf_input": false
},
"azure_ai/grok-3-mini": {
"deprecation_date": "2026-05-01",
@ -67390,7 +67600,8 @@
"output_cost_per_token": 1.27e-06,
"litellm_provider": "azure_ai",
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_pdf_input": false
},
"azure_ai/grok-4-fast-non-reasoning": {
"deprecation_date": "2026-05-01",
@ -67398,7 +67609,8 @@
"output_cost_per_token": 5e-07,
"litellm_provider": "azure_ai",
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_pdf_input": false
},
"azure_ai/grok-4-fast-reasoning": {
"deprecation_date": "2026-05-01",
@ -67406,7 +67618,8 @@
"output_cost_per_token": 5e-07,
"litellm_provider": "azure_ai",
"mode": "chat",
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_pdf_input": false
},
"bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": {
"input_cost_per_token": 7.2e-08,
@ -73076,6 +73289,7 @@
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_regions": ["global"],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
@ -73691,6 +73905,7 @@
"output_cost_per_token": 3.5e-06,
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true
@ -74745,6 +74960,7 @@
"text"
],
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": false,
"supports_reasoning": true,
"supports_tool_choice": true,
@ -74993,6 +75209,7 @@
"output_cost_per_token_above_200k_tokens": 1.2e-05,
"source": "https://docs.x.ai/developers/models/grok-4.7",
"supports_function_calling": true,
"supports_pdf_input": false,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,

29
litellm/provider_endpoints_support_backup.json
View file

@ -3724,7 +3724,7 @@
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"realtime": true,
"video_generations": false,
"batches": false,
"messages": false,
@ -4910,6 +4910,33 @@
"vector_stores_search": true
}
},
"opensearch": {
"display_name": "Amazon OpenSearch (`opensearch`)",
"url": "https://docs.litellm.ai/docs/providers/opensearch_vector_stores",
"endpoints": {
"chat_completions": false,
"responses": false,
"text_completion": false,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": false,
"interactions": false,
"systemone": false,
"a2a": false,
"generateContent": false,
"vector_stores_search": true
}
},
"helicone": {
"display_name": "Helicone (`helicone`)",
"url": "https://docs.litellm.ai/docs/providers/helicone",

8
litellm/proxy/batches_endpoints/litellm_executed_batches.py
View file

@ -35,7 +35,7 @@ from litellm.repositories.managed_batch_repository import ManagedBatchRepository
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import LiteLLMBatchCreateRequest, OpenAIFileObject, OpenAIFilesPurpose
from litellm.types.utils import LITELLM_EXECUTED_BATCH_PROVIDERS, ExtractedFileData, LiteLLMBatch, LlmProviders
from litellm.utils import resolve_api_key_secret_reference
from litellm.utils import aresolve_api_key_secret_reference
if TYPE_CHECKING:
from prisma import types as prisma_types
@ -227,7 +227,7 @@ async def upstream_lacks_files_api(api_base: str, api_key: str | None, http_clie
return response.status_code == httpx.codes.NOT_FOUND
def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
async def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str, str | None] | None:
model: Final = credentials.get("model")
api_base: Final = credentials.get("api_base")
api_key: Final = credentials.get("api_key")
@ -244,7 +244,7 @@ def _upstream_of(credentials: Mapping[str, object], provider: str) -> tuple[str,
return None
if resolved_api_base is None:
return None
return resolved_api_base, resolve_api_key_secret_reference(
return resolved_api_base, await aresolve_api_key_secret_reference(
resolved_api_key, model=model, llm_provider=provider, upstream=credentials
)
@ -255,7 +255,7 @@ async def litellm_executed_provider_for(
provider: Final = litellm_executed_provider_of(credentials)
if provider is None:
return None
upstream: Final = _upstream_of(credentials, provider)
upstream: Final = await _upstream_of(credentials, provider)
if upstream is None:
return None
return provider if await lacks_files_api(*upstream) else None

50
litellm/proxy/common_request_processing.py
View file

@ -129,6 +129,7 @@ from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _ar
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import (
endpoint_unsupported_by_model,
route_request,
)
from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
@ -871,6 +872,17 @@ _DECODED_JSON: Final = TypeAdapter(object)
_ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
def _fallback_probe_data(
pristine: object, fallback_model: str, key_aliases: object, group_alias: object
) -> Mapping[str, object]:
return {
**_REQUEST_FIELDS.validate_python(pristine),
"model": resolve_served_model(
fallback_model, _DECODED_JSON.validate_python(litellm.model_alias_map), key_aliases, group_alias
),
}
def _request_model(data: Mapping[str, object]) -> str | list[str] | None:
try:
return _REQUEST_MODEL.validate_python(data.get("model"), strict=True)
@ -2676,6 +2688,17 @@ class ProxyBaseLLMRequestProcessing:
original_model,
fallback_models,
)
raw_router_settings: Final = _DECODED_JSON.validate_python(
_REQUEST_FIELDS.validate_python(rate_limited_data).get("router_settings_override")
)
group_alias: Final = _DECODED_JSON.validate_python(
_REQUEST_FIELDS.validate_python(raw_router_settings).get("model_group_alias")
if isinstance(raw_router_settings, Mapping)
else None
)
key_aliases: Final[object] = cast( # cast-ok: auth aliases are dynamically typed at runtime
object, user_api_key_dict.aliases
)
try:
for fallback_model in fallback_models:
if fallback_model == original_model:
@ -2691,6 +2714,13 @@ class ProxyBaseLLMRequestProcessing:
self.data = independent_snapshot(pristine)
self.data["model"] = fallback_model
try:
if endpoint_unsupported_by_model(
route_type,
_fallback_probe_data(pristine, fallback_model, key_aliases, group_alias),
llm_router,
user_api_key_dict.team_id,
):
continue
return await self.common_processing_pre_call_logic(
request=request,
general_settings=general_settings,
@ -4029,13 +4059,13 @@ class ProxyBaseLLMRequestProcessing:
(assembled_response, cache_hit); the closure also runs
non-apply_guardrail post-call hooks via
_run_deferred_stream_guardrails.
- Every other anthropic_messages/aresponses stream gets a closure
that dispatches on the stored args shape, because the arming site
cannot tell the producers apart: native iterators store a single
ready-made logging coroutine to enqueue, while bridged streams
(LiteLLMCompletionStreamingIterator, and the plain SSE generator
AnthropicStreamWrapper returns for bridged /v1/messages) share
their inner CustomStreamWrapper's logging_obj and so store
- Every other anthropic_messages/aresponses/atext_completion stream
gets a closure that dispatches on the stored args shape, because the
arming site cannot tell the producers apart: native iterators store
a single ready-made logging coroutine to enqueue, while bridged
streams (LiteLLMCompletionStreamingIterator, TextCompletionStreamWrapper,
and the plain SSE generator AnthropicStreamWrapper returns for bridged
/v1/messages) share their inner CustomStreamWrapper's logging_obj and so store
(assembled_response, cache_hit); for those the closure only
dispatches success logging, matching the route's pre-existing
hook surface.
@ -4072,7 +4102,11 @@ class ProxyBaseLLMRequestProcessing:
logging_obj._on_deferred_stream_complete = _on_deferred_stream_complete
return
if route_type not in ("anthropic_messages", "aresponses") or not self._is_streaming_response(response):
if route_type not in (
"anthropic_messages",
"aresponses",
"atext_completion",
) or not self._is_streaming_response(response):
return
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER

5
litellm/proxy/common_utils/callback_utils.py
View file

@ -436,10 +436,7 @@ def get_model_group_from_litellm_kwargs(kwargs: dict) -> str | None:
def get_model_group_from_request_data(data: dict) -> str | None:
_metadata: Final = data.get("metadata", None) or {}
_model_group: Final = _metadata.get("model_group", None)
if _model_group is not None:
return _model_group
return None
return _model_group if isinstance(_model_group, str) else None
def get_remaining_tokens_and_requests_from_request_data(data: dict) -> dict[str, str]:

2
litellm/proxy/db/db_spend_update_writer.py
View file

@ -1900,8 +1900,6 @@ class DBSpendUpdateWriter:
proxy_logging_obj=proxy_logging_obj,
)
# NOTE: Daily tag spend is committed by a separate scheduler job.
################## Daily End-User Spend Update Transactions ##################
# Aggregate all in memory daily end-user spend transactions and commit to db
await self._flush_daily_spend_queue(

12
litellm/proxy/db/prisma_client.py
View file

@ -1127,16 +1127,16 @@ class PrismaManager:
return ProxyExtrasDBManager.build_request_log_indexes()
@staticmethod
def migration_data_check_report(use_migrate: bool = True) -> str | None:
"""The read-only pre-migration row check on its own, for `--check_migrations_against_existing_data`: the
refusal the migration step would print, or None when the pending schema changes
can be applied to the rows the database has."""
def migration_dry_run_summary(use_migrate: bool = True) -> tuple[bool, str]:
"""`--check_migrations_against_existing_data` on its own: the pending migrations applied inside a
transaction that is rolled back, as (whether they all applied, what the operator reads)."""
from litellm_proxy_extras.utils import (
ProxyExtrasDBManager, # pyright: ignore[reportUnknownVariableType] # litellm-proxy-extras is an optional install
)
from pydantic import TypeAdapter
refusal: Final[object] = ProxyExtrasDBManager.migration_data_check_report(use_migrate=use_migrate) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm-proxy-extras is an optional install
return refusal if isinstance(refusal, str) else None
summary: Final[object] = ProxyExtrasDBManager.migration_dry_run_summary(use_migrate=use_migrate) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # litellm-proxy-extras is an optional install
return TypeAdapter(tuple[bool, str]).validate_python(summary)
@staticmethod
def start_request_log_index_build() -> None:

36
litellm/proxy/guardrails/guardrail_hooks/grayswan/grayswan.py
View file

@ -82,6 +82,22 @@ def _as_plain_dict(item: object) -> Mapping[str, object]:
return cast("Mapping[str, object]", item) # cast-ok: wire rows are message/tool-call dicts
_STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object])
def _str_keyed(value: object) -> Mapping[str, object]:
return _STR_KEYED_MAPPING.validate_python(value) if isinstance(value, Mapping) else {}
def _request_route(request_data: Mapping[str, object]) -> str | None:
litellm_metadata: Final = request_data.get("litellm_metadata")
metadata_bucket: Final = _str_keyed(
litellm_metadata if litellm_metadata is not None else request_data.get("metadata")
)
route: Final = metadata_bucket.get("user_api_key_request_route")
return route if isinstance(route, str) else None
class GraySwanGuardrailMissingSecrets(Exception):
"""Raised when the Gray Swan API key is missing."""
@ -625,11 +641,8 @@ class GraySwanGuardrail(CustomGuardrail):
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
from litellm.llms import load_guardrail_translation_mappings
litellm_metadata: Final = request_data.get("litellm_metadata")
request_route: Final = (
litellm_metadata.get("user_api_key_request_route") if isinstance(litellm_metadata, Mapping) else None
)
route_call_types: Final = get_call_types_for_route(request_route) if isinstance(request_route, str) else None
request_route: Final = _request_route(request_data)
route_call_types: Final = get_call_types_for_route(request_route) if request_route is not None else None
call_type: Final = (
(route_call_types[0].value if route_call_types else None)
or (logging_obj.call_type if logging_obj is not None else None)
@ -698,13 +711,12 @@ class GraySwanGuardrail(CustomGuardrail):
inbound_headers: Final = self._extract_inbound_headers(request_data, logging_obj)
litellm_metadata: Final = request_data.get("litellm_metadata")
cleaned_litellm_metadata: Final = dict(litellm_metadata) if isinstance(litellm_metadata, dict) else {}
if inbound_headers:
existing_headers: Final = cleaned_litellm_metadata.get("headers")
cleaned_litellm_metadata["headers"] = (
{**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers
)
litellm_metadata: Final = _str_keyed(request_data.get("litellm_metadata"))
cleaned_litellm_metadata: Final = (
{**litellm_metadata, "headers": {**_str_keyed(litellm_metadata.get("headers")), **inbound_headers}}
if inbound_headers
else litellm_metadata
)
if cleaned_litellm_metadata:
sanitized: Final = _DECODED_JSON.validate_python(
safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={})

194
litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
View file

@ -12,11 +12,11 @@ import json
import os
import re
import time
from collections.abc import AsyncGenerator, Coroutine, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Mapping, Sequence
from dataclasses import dataclass, replace
from datetime import datetime
from re import Pattern
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, cast
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, cast
import yaml
from fastapi import HTTPException
@ -40,6 +40,7 @@ from litellm.proxy.guardrails.content_filter_data import (
)
from litellm.types.utils import (
CallTypes,
Delta,
Function,
GenericGuardrailAPIInputs,
GuardrailStatus,
@ -47,6 +48,9 @@ from litellm.types.utils import (
Message,
ModelResponse,
ModelResponseStream,
StreamingChoices,
TextChoices,
TextCompletionResponse,
)
if TYPE_CHECKING:
@ -78,6 +82,7 @@ from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern
MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1
_DECODED_ARGUMENTS: Final = TypeAdapter(object)
_StreamItemT: Final = TypeVar("_StreamItemT")
_JSON_ENTRIES: Final = TypeAdapter(list[object])
GAP_WORD_TOKENIZER: Final = re.compile(r"\b\w+\b")
SENTENCE_TERMINATORS: Final = re.compile(r"[.!?]+")
@ -139,6 +144,26 @@ class _CategoryConfigView(TypedDict):
category_file: str | None
def _streamed_choice_content(choice: StreamingChoices | TextChoices) -> str | None:
if isinstance(choice, TextChoices):
return choice.text if isinstance(choice.text, str) else ""
delta: Final = getattr(choice, "delta", None)
if not delta:
return None
content: Final = getattr(delta, "content", None)
return content if isinstance(content, str) else ""
def _streamed_choices(item: object) -> Sequence[StreamingChoices | TextChoices]:
if isinstance(item, (ModelResponseStream, TextCompletionResponse)):
return item.choices
return ()
def _is_choiceless_chunk(item: object) -> bool:
return isinstance(item, (ModelResponseStream, TextCompletionResponse)) and not item.choices
@dataclass(frozen=True, slots=True)
class _StreamedChoiceState:
buffered_text: str = ""
@ -146,6 +171,52 @@ class _StreamedChoiceState:
committed_detections: tuple[ContentFilterDetection, ...] = ()
latest_detections: tuple[ContentFilterDetection, ...] = ()
next_trim_len: int = 0
finished: bool = False
def _end_of_stream_chunk(
last_item: object, state_by_choice: Mapping[int, _StreamedChoiceState]
) -> ModelResponseStream | TextCompletionResponse | None:
unfinished: Final = tuple(index for index, state in state_by_choice.items() if not state.finished)
if not unfinished:
return None
if isinstance(last_item, TextCompletionResponse):
return TextCompletionResponse(
id=last_item.id,
created=last_item.created,
model=last_item.model,
object="text_completion",
choices=[TextChoices(index=index, text="") for index in unfinished],
).model_copy(update={"usage": None})
if isinstance(last_item, ModelResponseStream):
return ModelResponseStream(
id=last_item.id,
created=last_item.created,
model=last_item.model,
choices=[StreamingChoices(index=index, delta=Delta(content="")) for index in unfinished],
)
return None
async def _stream_then_end_of_stream_chunk(
response: AsyncIterable[_StreamItemT], end_of_stream_chunk: Callable[[_StreamItemT], _StreamItemT | None]
) -> AsyncGenerator[tuple[_StreamItemT, bool], None]:
last_item: _StreamItemT | None = None # rebind-ok: the end-of-stream chunk copies the last chunk's id and model
held_items: tuple[_StreamItemT, ...] = () # rebind-ok: the usage chunk must stay after the end-of-stream chunk
async for item in response:
last_item = item
if _is_choiceless_chunk(item):
held_items = (*held_items, item)
continue
for held_item in held_items:
yield held_item, False
held_items = ()
yield item, False
closing_item: Final = None if last_item is None else end_of_stream_chunk(last_item)
if closing_item is not None:
yield closing_item, True
for held_item in held_items:
yield held_item, False
@dataclass(frozen=True, slots=True)
@ -2104,72 +2175,75 @@ class ContentFilterGuardrail(CustomGuardrail):
)
try:
async for item in response:
if isinstance(item, ModelResponseStream) and item.choices:
for choice in item.choices:
if not (hasattr(choice, "delta") and choice.delta):
continue
async for item, stream_ended in _stream_then_end_of_stream_chunk(
response, lambda last_item: _end_of_stream_chunk(last_item, state_by_choice)
):
for choice in _streamed_choices(item):
new_content = _streamed_choice_content(choice)
if new_content is None:
continue
choice_index = getattr(choice, "index", 0)
if not isinstance(choice_index, int):
choice_index = 0
choice_index = getattr(choice, "index", 0)
if not isinstance(choice_index, int):
choice_index = 0
content = getattr(choice.delta, "content", None)
is_final = bool(getattr(choice, "finish_reason", None))
new_content = content if isinstance(content, str) else ""
if not new_content and not is_final:
continue
is_final = stream_ended or bool(getattr(choice, "finish_reason", None))
if not new_content and not is_final:
continue
previous_state = state_by_choice.get(choice_index, _StreamedChoiceState())
buffered_text = previous_state.buffered_text + new_content
if not buffered_text:
continue
previous_state = state_by_choice.get(choice_index, _StreamedChoiceState())
buffered_text = previous_state.buffered_text + new_content
if not buffered_text:
continue
# Add a space at the end if it's the final chunk to trigger word boundaries (\b)
text_to_scan = buffered_text + (" " if is_final else "")
choice_detections: list[ContentFilterDetection] = []
scan_started = time.perf_counter()
# Add a space at the end if it's the final chunk to trigger word boundaries (\b)
text_to_scan = buffered_text + (" " if is_final else "")
choice_detections: list[ContentFilterDetection] = []
scan_started = time.perf_counter()
try:
masked_text = self._filter_single_text(text_to_scan, detections=choice_detections)
if is_final and masked_text.endswith(" "):
masked_text = masked_text[:-1]
latest_detections = tuple(choice_detections)
except HTTPException:
state_by_choice[choice_index] = replace(
previous_state, latest_detections=tuple(choice_detections)
)
raise
except Exception as e:
verbose_proxy_logger.error("ContentFilterGuardrail: Error in masking: %s", e)
masked_text = text_to_scan # Fallback to current text
latest_detections = previous_state.latest_detections
finally:
scan_seconds += time.perf_counter() - scan_started
safe_to_yield_len = max(
previous_state.yielded_masked_text_len,
len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS),
try:
masked_text = self._filter_single_text(text_to_scan, detections=choice_detections)
if is_final and masked_text.endswith(" "):
masked_text = masked_text[:-1]
latest_detections = tuple(choice_detections)
except HTTPException:
state_by_choice[choice_index] = replace(
previous_state, latest_detections=tuple(choice_detections)
)
choice.delta.content = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len]
next_state = replace(
previous_state,
buffered_text=buffered_text,
yielded_masked_text_len=safe_to_yield_len,
latest_detections=latest_detections,
)
if is_final:
state_by_choice[choice_index] = next_state
continue
raise
except Exception as e:
verbose_proxy_logger.error("ContentFilterGuardrail: Error in masking: %s", e)
masked_text = text_to_scan # Fallback to current text
latest_detections = previous_state.latest_detections
finally:
scan_seconds += time.perf_counter() - scan_started
trim_started = time.perf_counter()
state_by_choice[choice_index] = self._trim_streamed_choice_buffer(next_state, masked_text, plan)
scan_seconds += time.perf_counter() - trim_started
safe_to_yield_len = max(
previous_state.yielded_masked_text_len,
len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS),
)
released_text = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len]
if isinstance(choice, TextChoices):
choice.text = released_text
else:
choice.delta.content = released_text
next_state = replace(
previous_state,
buffered_text=buffered_text,
yielded_masked_text_len=safe_to_yield_len,
latest_detections=latest_detections,
)
if is_final:
state_by_choice[choice_index] = replace(next_state, finished=True)
continue
yield item
else:
# Not a ModelResponseStream or no choices - yield as is
yield item
trim_started = time.perf_counter()
state_by_choice[choice_index] = self._trim_streamed_choice_buffer(next_state, masked_text, plan)
scan_seconds += time.perf_counter() - trim_started
if stream_ended and not any(_streamed_choice_content(choice) for choice in _streamed_choices(item)):
continue
yield item
except HTTPException:
status = "guardrail_intervened"
raise

5
litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py
View file

@ -647,9 +647,10 @@ class PromptSecurityGuardrail(CustomGuardrail):
async def _process_document_item(self, item: dict, user_api_key_alias: str | None) -> dict:
"""Process and sanitize document/file items."""
doc_data: Final = item.get("document") or item.get("file") or item
url_key: Final = "file_data" if isinstance(doc_data, dict) and not doc_data.get("url") else "url"
if isinstance(doc_data, dict):
url = doc_data.get("url", "")
url = doc_data.get(url_key) or ""
doc_content = doc_data.get("data", "")
else:
url = doc_data if isinstance(doc_data, str) else ""
@ -698,7 +699,7 @@ class PromptSecurityGuardrail(CustomGuardrail):
if url.startswith("data:") and header:
sanitized_url: Final = f"{header},{sanitized_encoded}"
if isinstance(doc_data, dict):
doc_data["url"] = sanitized_url
doc_data[url_key] = sanitized_url
elif isinstance(doc_data, dict):
doc_data["data"] = sanitized_encoded

2
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -865,7 +865,7 @@ async def _recorded_autorouter_savings(
entity_id_field="user_id",
entity_id=user_id,
exclude_entity_ids=None,
api_key=api_key,
api_key=(api_key,) if api_key is not None else None,
start_date=start_day,
end_date=end_day,
model=None,

168
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -23,6 +23,12 @@ from litellm.proxy.spend_tracking.key_metadata_recovery import (
)
from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attribution_enabled
from litellm.proxy.utils import PrismaClient
from litellm.repositories.daily_activity_buckets import (
api_key_bucket,
model_bucket,
model_group_bucket,
raw_api_key_filter,
)
from litellm.repositories.daily_activity_repository import DailyActivityRepository
from litellm.types.proxy.management_endpoints.common_daily_activity import (
BreakdownMetrics,
@ -182,7 +188,9 @@ class _AggregatedSpendData(TypedDict):
totals: ReadOnly[SpendMetrics]
def _key_metadata(api_key_metadata: Mapping[str, KeyMetadataRow], api_key: str) -> KeyMetadata:
def _key_metadata(api_key_metadata: Mapping[str, KeyMetadataRow], api_key: str | None) -> KeyMetadata:
if not api_key:
return KeyMetadata(key_alias="Unattributed", key_exists=False)
meta: Final = api_key_metadata.get(api_key)
return KeyMetadata(
key_alias=meta.key_alias if meta is not None else None,
@ -302,13 +310,14 @@ def update_breakdown_metrics(
parent bucket but never appear as an api_key row, and are kept out of the
per-request provider breakdown."""
api_key: Final = api_key_bucket(record.api_key)
is_ptu_sentinel: Final = record.api_key == PTU_SENTINEL_API_KEY
# A PTU sentinel row keys on the deployment id so a rename cannot move it, and carries
# the operator-facing name in model_group. The breakdown key is rendered directly as a
# label, so display the name; two deployments sharing one name merge here, which is
# what the write path used to do by collapsing them into a single row.
model_key: Final = (record.model_group or record.model) if is_ptu_sentinel else record.model
model_key: Final = (record.model_group or record.model) if is_ptu_sentinel else model_bucket(record.model)
# Update model breakdown
if model_key and model_key not in breakdown.models:
@ -321,18 +330,18 @@ def update_breakdown_metrics(
if not is_ptu_sentinel:
# Update API key breakdown for this model
if record.api_key not in breakdown.models[model_key].api_key_breakdown:
breakdown.models[model_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
if api_key not in breakdown.models[model_key].api_key_breakdown:
breakdown.models[model_key].api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.models[model_key].api_key_breakdown[record.api_key].metrics,
breakdown.models[model_key].api_key_breakdown[api_key].metrics = update_metrics(
breakdown.models[model_key].api_key_breakdown[api_key].metrics,
record,
)
# Update model group breakdown
model_group_key: Final = record.model_group or record.model
model_group_key: Final = model_group_bucket(record.model_group, record.model)
if model_group_key and model_group_key not in breakdown.model_groups:
breakdown.model_groups[model_group_key] = MetricWithMetadata(
metrics=SpendMetrics(),
@ -345,13 +354,13 @@ def update_breakdown_metrics(
if not is_ptu_sentinel:
# Update API key breakdown for this model
if record.api_key not in breakdown.model_groups[model_group_key].api_key_breakdown:
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
if api_key not in breakdown.model_groups[model_group_key].api_key_breakdown:
breakdown.model_groups[model_group_key].api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.model_groups[model_group_key].api_key_breakdown[record.api_key].metrics,
breakdown.model_groups[model_group_key].api_key_breakdown[api_key].metrics = update_metrics(
breakdown.model_groups[model_group_key].api_key_breakdown[api_key].metrics,
record,
)
@ -365,21 +374,20 @@ def update_breakdown_metrics(
breakdown.mcp_servers[record.mcp_namespaced_tool_name].metrics, record
)
# Update API key breakdown for this MCP server
if record.api_key not in breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown:
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key] = (
KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
if not is_ptu_sentinel:
# Update API key breakdown for this MCP server
if api_key not in breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown:
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[api_key] = (
KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
)
)
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[
record.api_key
].metrics = update_metrics(
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[record.api_key].metrics,
record,
)
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[api_key].metrics = update_metrics(
breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[api_key].metrics,
record,
)
if not is_ptu_sentinel:
# Update provider breakdown
@ -392,13 +400,13 @@ def update_breakdown_metrics(
breakdown.providers[provider].metrics = update_metrics(breakdown.providers[provider].metrics, record)
# Update API key breakdown for this provider
if record.api_key not in breakdown.providers[provider].api_key_breakdown:
breakdown.providers[provider].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
if api_key not in breakdown.providers[provider].api_key_breakdown:
breakdown.providers[provider].api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.providers[provider].api_key_breakdown[record.api_key].metrics,
breakdown.providers[provider].api_key_breakdown[api_key].metrics = update_metrics(
breakdown.providers[provider].api_key_breakdown[api_key].metrics,
record,
)
@ -413,25 +421,26 @@ def update_breakdown_metrics(
breakdown.endpoints[record.endpoint].metrics, record
)
# Update API key breakdown for this endpoint
if record.api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown:
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
if not is_ptu_sentinel:
# Update API key breakdown for this endpoint
if api_key not in breakdown.endpoints[record.endpoint].api_key_breakdown:
breakdown.endpoints[record.endpoint].api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.endpoints[record.endpoint].api_key_breakdown[api_key].metrics = update_metrics(
breakdown.endpoints[record.endpoint].api_key_breakdown[api_key].metrics,
record,
)
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.endpoints[record.endpoint].api_key_breakdown[record.api_key].metrics,
record,
)
if not is_ptu_sentinel:
# Update api key breakdown
if record.api_key not in breakdown.api_keys:
breakdown.api_keys[record.api_key] = KeyMetricWithMetadata(
if api_key not in breakdown.api_keys:
breakdown.api_keys[api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.api_keys[record.api_key].metrics = update_metrics(breakdown.api_keys[record.api_key].metrics, record)
breakdown.api_keys[api_key].metrics = update_metrics(breakdown.api_keys[api_key].metrics, record)
# Update entity-specific metrics if entity_id_field is provided
if entity_id_field:
@ -445,13 +454,13 @@ def update_breakdown_metrics(
if not is_ptu_sentinel:
# Update API key breakdown for this entity
if record.api_key not in breakdown.entities[entity_value].api_key_breakdown:
breakdown.entities[entity_value].api_key_breakdown[record.api_key] = KeyMetricWithMetadata(
if api_key not in breakdown.entities[entity_value].api_key_breakdown:
breakdown.entities[entity_value].api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = update_metrics(
breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics,
breakdown.entities[entity_value].api_key_breakdown[api_key].metrics = update_metrics(
breakdown.entities[entity_value].api_key_breakdown[api_key].metrics,
record,
)
@ -566,7 +575,7 @@ def daily_activity_scope(
entity_id_field: str,
entity_id: str | list[str] | None,
exclude_entity_ids: list[str] | None,
api_key: str | list[str] | None,
api_key: str | Sequence[str] | None,
start_date: str,
end_date: str,
model: str | None,
@ -577,9 +586,7 @@ def daily_activity_scope(
entity_ids: tuple[str, ...] | None = (
(entity_id,) if isinstance(entity_id, str) else tuple(entity_id) if entity_id is not None else None
)
api_keys: tuple[str, ...] | None = (
None if api_key in (None, "") else (api_key,) if isinstance(api_key, str) else tuple(api_key)
)
api_keys: Final = raw_api_key_filter(api_key)
return DailyActivityScope(
table=table_value,
entity_id_field=entity_id_field,
@ -715,6 +722,29 @@ _GROUP_DATE_ENDPOINT: Final = 62 # 0b0111110
_GROUP_DATE_ENDPOINT_API_KEY: Final = 30 # 0b0011110
def sum_spend_metrics(metrics: Sequence[SpendMetrics]) -> SpendMetrics:
return SpendMetrics(
spend=sum(metric.spend for metric in metrics),
flat_cost=sum(metric.flat_cost for metric in metrics),
ptu_hours=sum(metric.ptu_hours for metric in metrics),
prompt_tokens=sum(metric.prompt_tokens for metric in metrics),
completion_tokens=sum(metric.completion_tokens for metric in metrics),
cache_read_input_tokens=sum(metric.cache_read_input_tokens for metric in metrics),
cache_creation_input_tokens=sum(metric.cache_creation_input_tokens for metric in metrics),
compression_saved_tokens=sum(metric.compression_saved_tokens for metric in metrics),
compression_savings_spend=sum(metric.compression_savings_spend for metric in metrics),
prompt_caching_savings_spend=sum(metric.prompt_caching_savings_spend for metric in metrics),
gateway_injected_caching_savings_spend=sum(metric.gateway_injected_caching_savings_spend for metric in metrics),
autorouter_savings_spend=sum(metric.autorouter_savings_spend for metric in metrics),
total_tokens=sum(metric.total_tokens for metric in metrics),
successful_requests=sum(metric.successful_requests for metric in metrics),
failed_requests=sum(metric.failed_requests for metric in metrics),
api_requests=sum(metric.api_requests for metric in metrics),
total_response_time_ms=sum(metric.total_response_time_ms for metric in metrics),
timed_requests=sum(metric.timed_requests for metric in metrics),
)
def _record_to_spend_metrics(record: RollupMetricsRow) -> SpendMetrics:
"""Build a SpendMetrics directly from one already-aggregated rollup row.
@ -753,8 +783,7 @@ def _aggregate_grouping_sets_records_sync(
Each row carries a `group_level` bitmask (from Postgres GROUPING()) that
identifies which rollup level it belongs to. We dispatch the row's
pre-aggregated metrics straight into the matching bucket — no per-row
summing in Python and no nested update_metrics calls.
pre-aggregated metrics into the matching bucket, combining normalized names.
"""
total_metrics = SpendMetrics()
grouped_data: Final[dict[str, GroupedData]] = {}
@ -771,20 +800,23 @@ def _aggregate_grouping_sets_records_sync(
if existing is None:
target[key] = MetricWithMetadata(metrics=metrics, metadata={})
else:
existing.metrics = metrics
existing.metrics = sum_spend_metrics((existing.metrics, metrics))
def assign_api_key_breakdown(
target: dict[str, MetricWithMetadata],
parent_key: str,
api_key: str,
api_key: str | None,
metrics: SpendMetrics,
) -> None:
parent = target.get(parent_key)
if parent is None:
parent = MetricWithMetadata(metrics=SpendMetrics(), metadata={})
target[parent_key] = parent
parent.api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=metrics, metadata=_key_metadata(api_key_metadata, api_key)
bucket_key: Final = api_key_bucket(api_key)
previous: Final = parent.api_key_breakdown.get(bucket_key)
parent.api_key_breakdown[bucket_key] = KeyMetricWithMetadata(
metrics=sum_spend_metrics((previous.metrics, metrics)) if previous is not None else metrics,
metadata=_key_metadata(api_key_metadata, api_key),
)
for record in records:
@ -803,25 +835,23 @@ def _aggregate_grouping_sets_records_sync(
breakdown = ensure_date(record.date)["breakdown"]
if level == _GROUP_DATE_API_KEY:
if record.api_key and not is_ptu_sentinel:
breakdown.api_keys[record.api_key] = KeyMetricWithMetadata(
if not is_ptu_sentinel:
breakdown.api_keys[api_key_bucket(record.api_key)] = KeyMetricWithMetadata(
metrics=metrics,
metadata=_key_metadata(api_key_metadata, record.api_key),
)
elif level == _GROUP_DATE_MODEL:
if record.model:
assign_metric_with_metadata(breakdown.models, record.model, metrics)
assign_metric_with_metadata(breakdown.models, model_bucket(record.model), metrics)
elif level == _GROUP_DATE_MODEL_API_KEY:
if record.model and record.api_key and not is_ptu_sentinel:
assign_api_key_breakdown(breakdown.models, record.model, record.api_key, metrics)
if not is_ptu_sentinel:
assign_api_key_breakdown(breakdown.models, model_bucket(record.model), record.api_key, metrics)
elif level == _GROUP_DATE_MODEL_GROUP:
if record.model_group:
assign_metric_with_metadata(breakdown.model_groups, record.model_group, metrics)
assign_metric_with_metadata(breakdown.model_groups, model_group_bucket(record.model_group, None), metrics)
elif level == _GROUP_DATE_MODEL_GROUP_API_KEY:
if record.model_group and record.api_key and not is_ptu_sentinel:
if not is_ptu_sentinel:
assign_api_key_breakdown(
breakdown.model_groups,
record.model_group,
model_group_bucket(record.model_group, None),
record.api_key,
metrics,
)
@ -836,14 +866,14 @@ def _aggregate_grouping_sets_records_sync(
provider = record.custom_llm_provider or "unknown"
assign_metric_with_metadata(breakdown.providers, provider, provider_metrics)
elif level == _GROUP_DATE_PROVIDER_API_KEY:
if record.api_key and not is_ptu_sentinel:
if not is_ptu_sentinel:
provider = record.custom_llm_provider or "unknown"
assign_api_key_breakdown(breakdown.providers, provider, record.api_key, metrics)
elif level == _GROUP_DATE_MCP:
if record.mcp_namespaced_tool_name:
assign_metric_with_metadata(breakdown.mcp_servers, record.mcp_namespaced_tool_name, metrics)
elif level == _GROUP_DATE_MCP_API_KEY:
if record.mcp_namespaced_tool_name and record.api_key:
if record.mcp_namespaced_tool_name and not is_ptu_sentinel:
assign_api_key_breakdown(
breakdown.mcp_servers,
record.mcp_namespaced_tool_name,
@ -854,7 +884,7 @@ def _aggregate_grouping_sets_records_sync(
if record.endpoint:
assign_metric_with_metadata(breakdown.endpoints, record.endpoint, metrics)
elif level == _GROUP_DATE_ENDPOINT_API_KEY:
if record.endpoint and record.api_key:
if record.endpoint and not is_ptu_sentinel:
assign_api_key_breakdown(breakdown.endpoints, record.endpoint, record.api_key, metrics)
results: Final = [
@ -1013,8 +1043,8 @@ def _fold_entity_rollups_sync(
metrics = _record_to_spend_metrics(row)
if row.api_key_rolled:
bucket.metrics = metrics
elif row.api_key and row.api_key != PTU_SENTINEL_API_KEY:
bucket.api_key_breakdown[row.api_key] = KeyMetricWithMetadata(
elif row.api_key != PTU_SENTINEL_API_KEY:
bucket.api_key_breakdown[api_key_bucket(row.api_key)] = KeyMetricWithMetadata(
metrics=metrics, metadata=_key_metadata(api_key_metadata, row.api_key)
)

34
litellm/proxy/management_endpoints/daily_activity_routes.py
View file

@ -22,6 +22,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import (
parse_canonical_date_range,
raise_public,
spend_logs_window,
sum_spend_metrics,
)
from litellm.proxy.management_endpoints.daily_activity_scopes import (
AGENT_RESOLVER,
@ -38,6 +39,7 @@ from litellm.proxy.management_endpoints.ptu_consumption import single_team_id, w
from litellm.proxy.management_endpoints.team_endpoints import aggregated_date_range_error
from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
from litellm.proxy.utils import PrismaClient, get_prisma_client_or_throw
from litellm.repositories.daily_activity_buckets import api_key_bucket
from litellm.repositories.daily_activity_repository import DailyActivityRepository
from litellm.router import Router
from litellm.types.proxy.management_endpoints.common_daily_activity import (
@ -106,29 +108,9 @@ async def _resolved_scope(
return result
def _sum_metrics(metrics: Sequence[SpendMetrics]) -> SpendMetrics:
return SpendMetrics(
spend=sum(metric.spend for metric in metrics),
flat_cost=sum(metric.flat_cost for metric in metrics),
prompt_tokens=sum(metric.prompt_tokens for metric in metrics),
completion_tokens=sum(metric.completion_tokens for metric in metrics),
cache_read_input_tokens=sum(metric.cache_read_input_tokens for metric in metrics),
cache_creation_input_tokens=sum(metric.cache_creation_input_tokens for metric in metrics),
compression_saved_tokens=sum(metric.compression_saved_tokens for metric in metrics),
compression_savings_spend=sum(metric.compression_savings_spend for metric in metrics),
prompt_caching_savings_spend=sum(metric.prompt_caching_savings_spend for metric in metrics),
gateway_injected_caching_savings_spend=sum(metric.gateway_injected_caching_savings_spend for metric in metrics),
autorouter_savings_spend=sum(metric.autorouter_savings_spend for metric in metrics),
total_tokens=sum(metric.total_tokens for metric in metrics),
successful_requests=sum(metric.successful_requests for metric in metrics),
failed_requests=sum(metric.failed_requests for metric in metrics),
api_requests=sum(metric.api_requests for metric in metrics),
total_response_time_ms=sum(metric.total_response_time_ms for metric in metrics),
timed_requests=sum(metric.timed_requests for metric in metrics),
)
def _key_metadata(api_key: str, metadata: Mapping[str, KeyMetadataRow]) -> KeyMetadata:
if not api_key:
return KeyMetadata(key_alias="Unattributed", key_exists=False)
row: Final[KeyMetadataRow | None] = metadata.get(api_key)
if row is None:
return KeyMetadata()
@ -143,7 +125,7 @@ def _key_metadata(api_key: str, metadata: Mapping[str, KeyMetadataRow]) -> KeyMe
def _key_activity_row(row: KeySpendRow, metadata: Mapping[str, KeyMetadataRow]) -> KeySpendActivityRow:
return KeySpendActivityRow(
api_key=row.api_key,
api_key=api_key_bucket(row.api_key),
metrics=KeySpendMetrics(
spend=row.spend,
prompt_tokens=row.prompt_tokens,
@ -168,7 +150,7 @@ async def _key_activity_rows(
frozenset((resolved_scope.scope.start_date, resolved_scope.scope.end_date))
)
metadata: Final[Mapping[str, KeyMetadataRow]] = await repository.key_metadata(
frozenset(row.api_key for row in rows),
frozenset(row.api_key for row in rows if row.api_key),
spend_window,
)
return [_key_activity_row(row, metadata) for row in rows]
@ -231,11 +213,11 @@ def _fold_key_metrics(api_key: str, response: SpendAnalyticsPaginatedResponse) -
metadata: Final[KeyMetadata] = next(
day.breakdown.api_keys[api_key].metadata for day in response.results if api_key in day.breakdown.api_keys
)
return KeyActivityRow(api_key=api_key, metrics=_sum_metrics(metrics), metadata=metadata)
return KeyActivityRow(api_key=api_key, metrics=sum_spend_metrics(metrics), metadata=metadata)
def _search_rows(keys: Sequence[str], response: SpendAnalyticsPaginatedResponse) -> list[KeyActivityRow]:
return [row for key in keys if (row := _fold_key_metrics(key, response)) is not None]
return [row for key in keys if (row := _fold_key_metrics(api_key_bucket(key), response)) is not None]
def _csv_cell(value: object) -> object:

9
litellm/proxy/management_endpoints/daily_activity_scopes.py
View file

@ -17,6 +17,7 @@ from litellm.proxy.management_endpoints.organization_endpoints import resolve_or
from litellm.proxy.management_endpoints.tag_management_endpoints import get_tag_daily_activity_api_key_filter
from litellm.proxy.management_endpoints.team_endpoints import resolve_team_daily_activity_scope
from litellm.proxy.utils import PrismaClient
from litellm.repositories.daily_activity_buckets import raw_api_key_filter
from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable
@ -77,13 +78,7 @@ def _build_scope(
) -> ResolvedScope:
start_date: Final[str] = query.start_date or ""
end_date: Final[str] = query.end_date or ""
api_keys: Final[tuple[str, ...] | None] = (
None
if api_key_filter is None or api_key_filter == ""
else (api_key_filter,)
if isinstance(api_key_filter, str)
else tuple(api_key_filter)
)
api_keys: Final = raw_api_key_filter(api_key_filter)
return ResolvedScope(
scope=DailyActivityScope(
table=resolver.table,

8
litellm/proxy/management_endpoints/tag_management_endpoints.py
View file

@ -31,6 +31,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import (
get_daily_activity,
)
from litellm.proxy.management_helpers.utils import handle_budget_for_entity
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.repositories.model_repository import ModelRepository
from litellm.repositories.table_repositories import (
DailyTagSpendRepository,
@ -198,16 +199,17 @@ async def get_tag_daily_activity_api_key_filter(
user_api_key_dict: UserAPIKeyAuth,
requested_api_key: str | None,
) -> str | list[str] | None:
selected_key: Final = api_key_from_bucket(requested_api_key) if requested_api_key else None
user_role: Final = user_api_key_dict.user_role
if user_api_key_has_admin_view(user_api_key_dict) or (user_role is None or not user_role.is_internal_user_role):
return requested_api_key
return [selected_key] if selected_key is not None else None
scoped_api_keys: Final = await _get_internal_user_api_keys(
prisma_client=prisma_client,
user_api_key_dict=user_api_key_dict,
)
if requested_api_key is not None:
return requested_api_key if requested_api_key in scoped_api_keys else []
if selected_key is not None:
return [selected_key] if selected_key in scoped_api_keys else []
return scoped_api_keys

8
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -226,6 +226,7 @@ from litellm.proxy.management_helpers.utils import (
from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.chunked_in import delete_many_in, find_many_in
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import (
@ -7329,11 +7330,12 @@ async def resolve_team_daily_activity_scope(
if not user_api_keys:
user_api_keys = []
selected_key: Final = api_key_from_bucket(api_key) if api_key else None
final_api_key_filter: str | list[str] | None
if user_api_keys is None:
final_api_key_filter = api_key
elif api_key:
final_api_key_filter = api_key if api_key in user_api_keys else []
final_api_key_filter = [selected_key] if selected_key is not None else None
elif selected_key is not None:
final_api_key_filter = [selected_key] if selected_key in user_api_keys else []
else:
final_api_key_filter = user_api_keys

9
litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py
View file

@ -12,6 +12,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
)
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.secret_managers.main import (
GOOGLE_OIDC_SECRET_PREFIX,
UnresolvableSecretReference,
get_secret_str,
resolve_local_oidc_reference,
@ -47,6 +48,14 @@ def _credential_identity(credentials: VERTEX_CREDENTIALS_TYPES | None) -> str |
def _resolve_oidc_reference(credential: str | None) -> str | None:
if credential is None:
return None
if credential.startswith(GOOGLE_OIDC_SECRET_PREFIX):
raise HTTPException(
status_code=401,
detail=(
"pass-through credential oidc/google secret reference could not be resolved: pass-through routes "
"do not fetch Google ID tokens. Use a static key, oidc/file/<absolute path> or oidc/env/<VARIABLE_NAME>."
),
)
match resolve_local_oidc_reference(credential):
case UnresolvableSecretReference(kind=kind, reason=reason):
raise HTTPException(

18
litellm/proxy/proxy_cli.py
View file

@ -276,18 +276,19 @@ class ProxyInitializationHelpers:
@staticmethod
def run_migration_data_check(use_migrate: bool) -> int:
"""`--check_migrations_against_existing_data`: the pre-migration row check against DATABASE_URL, nothing applied.
0 when the pending schema changes fit the rows, 2 with the refusal when they do not."""
"""`--check_migrations_against_existing_data`: the pending migrations applied to DATABASE_URL inside one
transaction that is rolled back, so Postgres reports what the upgrade would hit. 0 when they all apply,
2 with Postgres's error when one does not."""
if os.getenv("DATABASE_URL") is None:
raise click.UsageError("--check_migrations_against_existing_data requires DATABASE_URL")
from litellm.proxy.db.prisma_client import PrismaManager
refusal: Final[str | None] = PrismaManager.migration_data_check_report(use_migrate=use_migrate) # pyright: ignore[reportUnknownVariableType] # litellm-proxy-extras is an optional install
if refusal is None:
print("LiteLLM Proxy: the pending migrations and indexes can be applied to the rows this database has.")
ok, summary = PrismaManager.migration_dry_run_summary(use_migrate=use_migrate)
if ok:
print(f"LiteLLM Proxy: {summary}")
return 0
print(
f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {refusal}\033[0m", file=sys.stderr, flush=True
f"\033[1;31mLiteLLM Proxy: Database migration cannot proceed. {summary}\033[0m", file=sys.stderr, flush=True
)
return 2
@ -1059,8 +1060,9 @@ class ProxyInitializationHelpers:
"--check_migrations_against_existing_data",
is_flag=True,
default=False,
help="Check, read-only, that the rows in DATABASE_URL can take this version's pending migrations and indexes, "
"print what would block them, then exit without applying anything. Exit code 2 when something blocks.",
help="Dry-run this version's pending migrations on DATABASE_URL: the real SQL inside one transaction that is "
"rolled back, so Postgres reports the row or schema that would make the upgrade fail, then exit. Holds the same "
"locks the upgrade will. Exit code 2 when a migration fails.",
)
@click.option(
"--keepalive_timeout",

62
litellm/proxy/proxy_server.py
View file

@ -79,6 +79,7 @@ from litellm.constants import (
DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL,
DEFAULT_SHARED_HEALTH_CHECK_TTL,
DEFAULT_SLACK_ALERTING_THRESHOLD,
GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS,
INTERNAL_CALLBACK_NAMES,
LANGFUSE_SHUTDOWN_FLUSH_TIMEOUT_MILLIS,
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS,
@ -98,6 +99,7 @@ from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,
initialize_sentry,
)
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.token_counter import (
@ -207,7 +209,7 @@ from litellm.types.utils import (
is_server_derived_pricing_key,
pricing_override_fields,
)
from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential
from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential
if TYPE_CHECKING:
from aiohttp import ClientSession
@ -1274,15 +1276,44 @@ async def _flush_spend_logs_queue_on_shutdown() -> None:
async def _flush_spend_on_shutdown() -> None:
from litellm.proxy.utils import update_daily_tag_spend
await flush_spend_counters_on_shutdown()
if prisma_client is not None:
await update_daily_tag_spend(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
await _flush_spend_logs_queue_on_shutdown()
async def _flush_spend_around_logging_drain_on_shutdown(
flush_spend: Callable[[], Awaitable[None]] = _flush_spend_on_shutdown,
*,
worker: LoggingWorker = GLOBAL_LOGGING_WORKER,
clock: Callable[[], float] = time.monotonic,
drain_tasks: Callable[[float], Awaitable[None]] = drain_logging_tasks,
) -> None:
await flush_spend()
await drain_logging_tasks()
remaining: Final = GracefulShutdownManager.remaining(clock=clock)
callback_timeout: Final = max(0.0, remaining - GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS)
verbose_proxy_logger.info(
"Draining success callbacks before the final spend flush remaining_s=%.3f callback_timeout_s=%.3f",
remaining,
callback_timeout,
)
async def drain_callbacks() -> None:
await drain_tasks(callback_timeout)
await worker.flush()
try:
await asyncio.wait_for(drain_callbacks(), timeout=callback_timeout)
except asyncio.TimeoutError:
verbose_proxy_logger.warning(
"Success callback shutdown budget exhausted with %d unfinished callbacks; accounting may be incomplete",
worker.pending_count,
)
finally:
await worker.stop(discard_pending=True, timeout=GracefulShutdownManager.remaining(2.0, clock=clock))
await drain_tasks(GracefulShutdownManager.remaining(5.0, clock=clock))
await flush_spend()
@ -1445,6 +1476,13 @@ async def _connect_to_count_stored_values() -> SupportsRawQueries:
@asynccontextmanager
async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]:
with GracefulShutdownManager.capture_sigterm():
async with _proxy_startup_event(app) as state:
yield state
@asynccontextmanager
async def _proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState, None]:
global \
prisma_client, \
master_key, \
@ -14764,10 +14802,10 @@ def _provider_count_outcome(
return result
def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
async def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
call_params: Final = _LITELLM_PARAMS_COPY.validate_python(litellm_params)
try:
load_credentials_from_list(call_params)
return {**call_params, **(await acredential_updates(call_params))}
except litellm.AuthenticationError as e:
raise ProxyException(
message=e.message,
@ -14775,13 +14813,19 @@ def _with_resolved_api_key(litellm_params: object) -> Mapping[str, object]:
param="model",
code=401,
) from e
return call_params
except litellm.ServiceUnavailableError as e:
raise ProxyException(
message=e.message,
type="service_unavailable",
param="model",
code=503,
) from e
def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None:
async def _with_resolved_credentials(deployment: Mapping[str, object] | None) -> dict[str, object] | None:
if deployment is None:
return None
return {**deployment, "litellm_params": _with_resolved_api_key(deployment.get("litellm_params", {}))}
return {**deployment, "litellm_params": await _with_resolved_api_key(deployment.get("litellm_params", {}))}
_DEPLOYMENT_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
@ -14844,7 +14888,7 @@ async def _try_provider_token_count(
model_to_use=model_to_use or "",
messages=messages,
contents=contents,
deployment=_with_resolved_credentials(deployment_in_region),
deployment=await _with_resolved_credentials(deployment_in_region),
request_model=request_model,
tools=tools,
system=system,
@ -14878,7 +14922,7 @@ async def _try_native_responses_count(
model_to_use=model_to_use,
request=responses_request,
policy=policy,
deployment=_with_resolved_credentials(deployment),
deployment=await _with_resolved_credentials(deployment),
request_model=request_model,
)
)

2
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -3612,7 +3612,7 @@
"key": "api_key",
"label": "Soniox API Key",
"placeholder": null,
"tooltip": "Currently only the async Speech-to-Text REST API (api.soniox.com) is supported. Realtime STT (stt-rt.soniox.com) and TTS (tts-rt.soniox.com) are not yet available.",
"tooltip": "Supports async Speech-to-Text (api.soniox.com) on /v1/audio/transcriptions and realtime Speech-to-Text (stt-rt.soniox.com) on /v1/realtime. TTS (tts-rt.soniox.com) is not yet available.",
"required": true,
"field_type": "password",
"options": null,

6
litellm/proxy/rag_endpoints/endpoints.py
View file

@ -54,6 +54,7 @@ from litellm.proxy.rag_endpoints.upload_security import (
from litellm.proxy.vector_store_endpoints.endpoints import (
build_request_data_from_managed_vector_store,
reject_caller_embedding_selection_params,
reject_caller_opensearch_index_name,
)
from litellm.proxy.vector_store_endpoints.utils import (
assert_user_can_access_vector_store_id,
@ -913,6 +914,11 @@ async def rag_query(
**retrieval_config,
**store_data,
}
reject_caller_opensearch_index_name(
caller_body=retrieval_config,
request_fields=merged_retrieval_config,
source="retrieval_config",
)
effective_vector_store_config: Final = (
managed_store_params
if managed_store is not None

16
litellm/proxy/response_api_endpoints/endpoints.py
View file

@ -34,6 +34,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
from litellm.caching.dual_cache import DualCache
from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
@ -2086,6 +2087,14 @@ def _routing_hints_from_first_ws_frame(first_message: str) -> Mapping[str, objec
def _responses_ws_failure_frame(failure: Exception) -> str:
if isinstance(failure, EndpointNotSupportedByModelError):
return json.dumps(
{
"type": "error",
"status": 400,
"error": {"type": "invalid_request_error", "message": failure.message.split("\n\n", 1)[0]},
}
)
proxy_exception: Final = handle_exception_on_proxy(failure)
status: Final = int(proxy_exception.code) if str(proxy_exception.code).isdigit() else 500
error_type: Final = (
@ -2444,7 +2453,12 @@ async def _responses_websocket_session(
with contextlib.suppress(Exception):
await websocket.send_text(_responses_ws_failure_frame(e))
with contextlib.suppress(Exception):
await websocket.close(code=1011, reason="Internal server error")
await websocket.close(
code=1008 if isinstance(e, EndpointNotSupportedByModelError) else 1011,
reason="Pre-call error"
if isinstance(e, EndpointNotSupportedByModelError)
else "Internal server error",
)
return crash_row
return bill_socket_failure(outcome) if isinstance(outcome, Exception) else None

22
litellm/proxy/route_llm_request.py
View file

@ -431,6 +431,28 @@ def raise_if_required_body_param_missing(
)
def endpoint_unsupported_by_model(
route_type: str,
data: Mapping[str, object],
llm_router: LitellmRouter | None,
team_id: str | None,
) -> bool:
model: Final = data.get("model")
if not isinstance(model, str) or llm_router is None:
return False
metadata: Final = data.get("metadata")
request_kwargs: Final = {
**data,
"metadata": {
**(metadata if isinstance(metadata, Mapping) else {}),
"user_api_key_team_id": team_id,
},
}
return bool(llm_router.get_routable_deployments(model, request_kwargs)) and not llm_router.get_routable_deployments(
model, request_kwargs, call_type=route_type
)
class MockTestingParamsDisabledError(HTTPException):
def __init__(self, params: tuple[str, ...]):
super().__init__(

58
litellm/proxy/shutdown/graceful_shutdown_manager.py
View file

@ -5,7 +5,8 @@ Kubernetes terminates a pod by sending ``SIGTERM`` and, after
``terminationGracePeriodSeconds``, ``SIGKILL``. On ``SIGTERM`` uvicorn stops
accepting connections and waits up to ``GRACEFUL_SHUTDOWN_TIMEOUT`` for open
ones, cancelling whatever is still running, before the lifespan teardown that
runs this drain.
runs this drain. The lifespan registers a SIGTERM wrapper before serving
requests so callback draining can subtract the preceding connection drain.
A fixed ``preStop`` sleep can not solve this: it has to be sized for the
*worst-case* request, so it either wastes time on every routine shutdown or is
@ -20,8 +21,12 @@ granularity of ``InFlightRequestsMiddleware``.
import asyncio
import os
import signal
import threading
import time
from collections.abc import Callable
from collections.abc import Callable, Generator
from contextlib import contextmanager
from types import FrameType
from typing import Final
from litellm._logging import verbose_proxy_logger
@ -44,6 +49,7 @@ class GracefulShutdownManager:
_is_shutting_down: bool = False
_shutdown_started_at: float | None = None
_drain_performed: bool = False
_sigterm_handler: Callable[[int, FrameType | None], None] | None = None
@classmethod
def is_shutting_down(cls) -> bool:
@ -71,7 +77,7 @@ class GracefulShutdownManager:
return DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT
@classmethod
def start_shutdown(cls) -> None:
def start_shutdown(cls, clock: Callable[[], float] = time.monotonic) -> None:
"""
Mark the worker as draining. Idempotent — repeated calls (e.g. SIGTERM
followed by a preStop hit on /health/drain) do not reset the clock.
@ -79,12 +85,42 @@ class GracefulShutdownManager:
if cls._is_shutting_down:
return
cls._is_shutting_down = True
cls._shutdown_started_at = time.monotonic()
cls._shutdown_started_at = clock()
verbose_proxy_logger.info(
"graceful_shutdown_started in_flight_requests=%s",
get_in_flight_requests(),
)
@classmethod
def remaining(cls, cap: float | None = None, *, clock: Callable[[], float] = time.monotonic) -> float:
elapsed: Final = 0.0 if cls._shutdown_started_at is None else clock() - cls._shutdown_started_at
remaining: Final = max(0.0, cls.get_timeout() - elapsed)
return remaining if cap is None else min(max(0.0, cap), remaining)
@classmethod
@contextmanager
def capture_sigterm(cls, clock: Callable[[], float] = time.monotonic) -> Generator[None, None, None]:
if threading.current_thread() is not threading.main_thread():
yield
return
previous: Final = signal.getsignal(signal.SIGTERM)
if not callable(previous) or previous is cls._sigterm_handler:
yield
return
def handle_sigterm(signum: int, frame: FrameType | None) -> None:
cls.start_shutdown(clock)
previous(signum, frame)
cls._sigterm_handler = handle_sigterm
signal.signal(signal.SIGTERM, handle_sigterm)
try:
yield
finally:
if signal.getsignal(signal.SIGTERM) is handle_sigterm:
signal.signal(signal.SIGTERM, previous)
cls._sigterm_handler = None
@classmethod
async def wait_for_drain(
cls,
@ -93,13 +129,14 @@ class GracefulShutdownManager:
count_fn: Callable[[], int] | None = None,
poll_interval: float = _DRAIN_POLL_INTERVAL,
log_interval: float = _DRAIN_LOG_INTERVAL,
clock: Callable[[], float] = time.monotonic,
) -> int:
"""
Poll the in-flight request counter until it reaches the drain target or
``timeout`` seconds elapse.
Args:
timeout: Max seconds to wait. Defaults to ``get_timeout()``.
timeout: Max seconds to wait. Defaults to the remaining shutdown budget.
exclude_self: When the caller is itself an in-flight HTTP request
(the /health/drain endpoint), set this so the caller's own
request is not counted as outstanding work.
@ -107,6 +144,7 @@ class GracefulShutdownManager:
live ``InFlightRequestsMiddleware`` counter; injectable for tests.
poll_interval: Seconds between counter polls.
log_interval: Minimum seconds between ``drain_waiting`` log lines.
clock: Monotonic clock shared with the shutdown timestamp.
Returns:
Number of requests that drained while waiting (>= 0).
@ -120,7 +158,7 @@ class GracefulShutdownManager:
cls._drain_performed = True
if timeout is None:
timeout = cls.get_timeout()
timeout = cls.remaining(clock=clock)
if count_fn is None:
count_fn = get_in_flight_requests
@ -128,7 +166,7 @@ class GracefulShutdownManager:
# and so counts itself; treat <=1 as "drained" in that case.
target: Final = 1 if exclude_self else 0
start: Final = time.monotonic()
start: Final = clock()
initial: Final = count_fn()
last_log = start
@ -142,11 +180,11 @@ class GracefulShutdownManager:
verbose_proxy_logger.info(
"graceful_shutdown_complete drained_requests=%s elapsed_s=%.2f",
drained,
time.monotonic() - start,
clock() - start,
)
return drained
elapsed = time.monotonic() - start
elapsed = clock() - start
if elapsed >= timeout:
verbose_proxy_logger.warning(
"graceful_shutdown_timeout in_flight_requests=%s elapsed_s=%.2f "
@ -157,7 +195,7 @@ class GracefulShutdownManager:
)
return max(0, initial - current)
now = time.monotonic()
now = clock()
if now - last_log >= log_interval:
verbose_proxy_logger.info(
"drain_waiting in_flight_requests=%s elapsed_s=%.2f",

8
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -464,7 +464,11 @@ def _sl_attribution_fallback(
) -> str:
if standard_logging_payload is None:
return ""
return standard_logging_payload.get(field) or ""
return _string_or_empty(standard_logging_payload.get(field))
def _string_or_empty(value: object) -> str:
return value if isinstance(value, str) else ""
def _deployment_provider(deployment: DeploymentTypedDict) -> str | None:
@ -682,7 +686,7 @@ def get_logging_payload(
_model_id: Final = metadata.get("model_info", {}).get("id", "") or _sl_attribution_fallback(
standard_logging_payload, "model_id"
)
_model_group: Final = metadata.get("model_group", "") or _sl_attribution_fallback(
_model_group: Final = _string_or_empty(metadata.get("model_group")) or _sl_attribution_fallback(
standard_logging_payload, "model_group"
)
_api_base: Final = litellm_params.get("api_base", "") or _sl_attribution_fallback(

26
litellm/proxy/vector_store_endpoints/endpoints.py
View file

@ -17,6 +17,7 @@ from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.common_utils.http_parsing_utils import read_raw_json_body
from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER
from litellm.proxy.utils import jsonify_object
from litellm.proxy.vector_store_endpoints.utils import (
assert_proxy_admin_for_vector_store_index_management,
@ -24,6 +25,7 @@ from litellm.proxy.vector_store_endpoints.utils import (
get_litellm_managed_vector_store,
)
from litellm.repositories.table_repositories import ManagedVectorStoreIndexRepository
from litellm.types.utils import LlmProviders
from litellm.types.vector_stores import IndexCreateRequest, IndexListResponse
from litellm.vector_stores.utils import VectorStoreWriteBody, callers_metadata_snapshot
from litellm.vector_stores.vector_store_registry import VectorStoreIndexRegistry
@ -52,6 +54,24 @@ def reject_caller_embedding_selection_params(payload: Mapping[str, object], sour
)
def reject_caller_opensearch_index_name(
caller_body: Mapping[str, object], request_fields: Mapping[str, object], source: str
) -> None:
provider: Final = request_fields.get("custom_llm_provider")
if (
isinstance(provider, str)
and provider.split("/", 1)[0] == LlmProviders.OPENSEARCH.value
and "opensearch_index_name" in caller_body
):
raise HTTPException(
status_code=400,
detail={
"error": f"'opensearch_index_name' cannot be set in {source}. "
"The index comes from the vector store's server-side registration."
},
)
########################################################
# OpenAI Compatible Endpoints
########################################################
@ -138,11 +158,17 @@ async def vector_store_search(
data = await read_request_body(request=request) # rebind-ok: pre-existing rebinding on a rename-only line
reject_caller_embedding_selection_params(payload=data, source="the search request body")
data["vector_store_id"] = vector_store_id
caller_body: Final = JSON_OBJECT_ADAPTER.validate_python(data)
# Check for legacy vector store registry (non-managed vector stores)
data = await _update_request_data_with_litellm_managed_vector_store_registry(
data=data, vector_store_id=vector_store_id, user_api_key_dict=user_api_key_dict
)
reject_caller_opensearch_index_name(
caller_body=caller_body,
request_fields=JSON_OBJECT_ADAPTER.validate_python(data),
source="the search request body",
)
# The managed_vector_stores pre-call hook will handle:
# 1. Decoding managed vector store IDs

2
litellm/realtime_api/README.md
View file

@ -4,6 +4,6 @@ Supported endpoints:
- WebSocket: `/v1/realtime` (with `intent=transcription` for transcription-only sessions)
- HTTP: `/v1/realtime/client_secrets`, `/v1/realtime/transcription_sessions`
Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI.
Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI, Gemini, Meta, Soniox.
For user-facing documentation and usage examples, see the litellm-docs repo.

18
litellm/realtime_api/main.py
View file

@ -7,6 +7,7 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast
from pydantic import TypeAdapter
from typing_extensions import ReadOnly, TypedDict, assert_never
import litellm
@ -34,8 +35,8 @@ from litellm.types.utils import CallTypes, LlmProviders
from litellm.utils import (
ModelInfoLookupMiss,
ProviderConfigManager,
aresolve_api_key_secret_reference,
lookup_model_info_for_registration,
resolve_api_key_secret_reference,
)
from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
@ -55,6 +56,7 @@ from ..llms.openai.realtime.handler import (
confirm_session_started,
)
from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async
from ..llms.soniox.realtime.transformation import SonioxRealtimeOptions, soniox_realtime_health_check
from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model
from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config
from ..llms.vertex_ai.vertex_llm_base import VertexBase
@ -75,6 +77,7 @@ vertex_llm_base: Final = VertexBase()
base_llm_http_handler = BaseLLMHTTPHandler()
_EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
_EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
_DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object])
def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]:
@ -527,6 +530,7 @@ async def _arealtime(
custom_llm_provider=_custom_llm_provider,
)
deployment_params: Final = _DEPLOYMENT_PARAMS.validate_python(kwargs)
if live_session_start is not None:
await _alive_session(
model=model,
@ -537,7 +541,7 @@ async def _arealtime(
api_base=dynamic_api_base or litellm_params.api_base,
api_key=dynamic_api_key,
litellm_metadata=_build_litellm_metadata(kwargs),
litellm_params=kwargs,
litellm_params=deployment_params,
)
return
@ -548,6 +552,7 @@ async def _arealtime(
model=model,
provider=LlmProviders(_custom_llm_provider),
drop_params=drop_params,
litellm_params=deployment_params,
)
if provider_config is not None:
await base_llm_http_handler.async_realtime(
@ -840,7 +845,7 @@ async def realtime_health_check(
resolved_api_base: Final = cast( # cast-ok: provider parameters expose optional string endpoints
str | None, api_base or resolved_params.get("api_base")
)
resolved_api_key: Final = resolve_api_key_secret_reference(
resolved_api_key: Final = await aresolve_api_key_secret_reference(
cast( # cast-ok: provider parameters expose optional string credentials
str | None, api_key or resolved_params.get("api_key")
),
@ -882,6 +887,13 @@ async def realtime_health_check(
)
elif custom_llm_provider == "openai":
url = openai_realtime.construct_url(api_base=openai_api_base, query_params={"model": model})
elif custom_llm_provider == "soniox":
return await soniox_realtime_health_check(
model=model,
api_base=resolved_api_base,
api_key=health_check_api_key,
options=SonioxRealtimeOptions.model_validate(dict(resolved_params)),
)
elif custom_llm_provider == "xai":
url = xai_realtime.construct_url(
api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model}

39
litellm/repositories/daily_activity_buckets.py Normal file
View file

@ -0,0 +1,39 @@
from collections.abc import Sequence
from typing import Final
from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL
UNKNOWN_MODEL_GROUP: Final = "unknown-model-group"
UNATTRIBUTED_API_KEY: Final = "unattributed"
_RAW_API_KEY_PREFIX: Final = "raw:"
def model_bucket(model: str | None) -> str:
return model or UNKNOWN_MODEL_SPEND_LOG_MODEL
def model_group_bucket(model_group: str | None, model: str | None) -> str:
return model_group or model or UNKNOWN_MODEL_GROUP
def api_key_bucket(api_key: str | None) -> str:
if not api_key:
return UNATTRIBUTED_API_KEY
if api_key == UNATTRIBUTED_API_KEY or api_key.startswith(_RAW_API_KEY_PREFIX):
return _RAW_API_KEY_PREFIX + api_key
return api_key
def api_key_from_bucket(api_key: str) -> str:
if api_key == UNATTRIBUTED_API_KEY:
return ""
return api_key.removeprefix(_RAW_API_KEY_PREFIX)
def raw_api_key_filter(api_key: str | Sequence[str] | None) -> tuple[str, ...] | None:
"""String filters are reporting IDs; sequences are raw keys resolved by authorization."""
if api_key is None or api_key == "":
return None
if isinstance(api_key, str):
return (api_key_from_bucket(api_key),)
return tuple(api_key)

9
litellm/repositories/daily_activity_repository.py
View file

@ -10,6 +10,7 @@ from typing_extensions import assert_never
from litellm import constants
from litellm._logging import verbose_proxy_logger
from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL
from litellm.repositories.chunked_in import find_many_in
from litellm.repositories.daily_activity_sql import (
ExportCursor,
@ -344,7 +345,13 @@ class DailyActivityRepository:
"date": {"gte": adjusted_start, "lte": adjusted_end},
**({scope.entity_id_field: {"in": list(scope.entity_ids)}} if scope.entity_ids is not None else {}),
**exclusion_filter,
**({"model": scope.model} if scope.model else {}),
**(
{"AND": [{"OR": [{"model": None}, {"model": ""}, {"model": scope.model}]}]}
if scope.model == UNKNOWN_MODEL_SPEND_LOG_MODEL
else {"model": scope.model}
if scope.model
else {}
),
**({"api_key": {"in": list(scope.api_keys)}} if scope.api_keys is not None else {}),
}
count, rows = await asyncio.gather(

60
litellm/repositories/daily_activity_sql.py
View file

@ -8,11 +8,18 @@ from typing import Final
from typing_extensions import assert_never
from litellm import constants
from litellm.constants import PTU_SENTINEL_API_KEY
from litellm.constants import PTU_SENTINEL_API_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL
from litellm.repositories.daily_activity_buckets import UNATTRIBUTED_API_KEY, UNKNOWN_MODEL_GROUP
from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable, ExportType
_API_KEY_ROLLED_UP_BIT: Final = 32
_MODEL_GROUP_EXPR: Final = "COALESCE(NULLIF(model_group, ''), model)"
_MODEL_EXPR: Final = f"COALESCE(NULLIF(model, ''), '{UNKNOWN_MODEL_SPEND_LOG_MODEL}')"
_MODEL_GROUP_EXPR: Final = f"COALESCE(NULLIF(model_group, ''), NULLIF(model, ''), '{UNKNOWN_MODEL_GROUP}')"
_API_KEY_EXPR: Final = (
f"CASE WHEN api_key = '' THEN '{UNATTRIBUTED_API_KEY}' "
f"WHEN api_key = '{UNATTRIBUTED_API_KEY}' OR api_key LIKE 'raw:%' THEN 'raw:' || api_key ELSE api_key END"
)
@dataclass(frozen=True, slots=True)
@ -86,7 +93,11 @@ def build_where_clause(scope: DailyActivityScope, *, start_index: int = 1) -> tu
if scope.exclude_entity_ids
else ()
),
*((f"model = ${model_index}",) if scope.model else ()),
*(
(f"{_MODEL_EXPR if scope.model == UNKNOWN_MODEL_SPEND_LOG_MODEL else 'model'} = ${model_index}",)
if scope.model
else ()
),
*(
("FALSE",)
if scope.api_keys == ()
@ -143,7 +154,7 @@ def _top_api_keys_sql(pg_table: str, where_clause: str, *, sentinel_param: int,
FROM "{pg_table}"
WHERE {where_clause} AND api_key <> ${sentinel_param}
GROUP BY api_key
ORDER BY SUM(spend::numeric) DESC, api_key
ORDER BY SUM(spend::numeric) DESC, {_API_KEY_EXPR}
LIMIT ${limit_param}
"""
@ -162,13 +173,13 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq
(SELECT
date,
NULL::text AS api_key,
model,
{_MODEL_EXPR} AS model,
{_MODEL_GROUP_EXPR} AS model_group,
custom_llm_provider,
mcp_namespaced_tool_name,
endpoint,
(GROUPING(date) << 6) | {_API_KEY_ROLLED_UP_BIT}
| GROUPING(model, {_MODEL_GROUP_EXPR},
| GROUPING({_MODEL_EXPR}, {_MODEL_GROUP_EXPR},
custom_llm_provider, mcp_namespaced_tool_name,
endpoint) AS group_level,
NULL::bigint AS distinct_api_keys,{metric_select}
@ -176,7 +187,7 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq
WHERE {where_clause}
GROUP BY GROUPING SETS (
(date),
(date, model),
(date, {_MODEL_EXPR}),
(date, {_MODEL_GROUP_EXPR}),
(date, custom_llm_provider),
(date, mcp_namespaced_tool_name),
@ -190,12 +201,12 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq
SELECT
date,
api_key,
model,
{_MODEL_EXPR} AS model,
{_MODEL_GROUP_EXPR} AS model_group,
custom_llm_provider,
mcp_namespaced_tool_name,
endpoint,
GROUPING(date, api_key, model, {_MODEL_GROUP_EXPR},
GROUPING(date, api_key, {_MODEL_EXPR}, {_MODEL_GROUP_EXPR},
custom_llm_provider, mcp_namespaced_tool_name,
endpoint) AS group_level,
MAX(top_api_keys.distinct_api_keys) AS distinct_api_keys,{metric_select}
@ -203,7 +214,7 @@ def build_aggregated_sql(scope: DailyActivityScope, *, api_key_limit: int) -> Sq
WHERE {where_clause}
GROUP BY GROUPING SETS (
(date, api_key),
(date, model, api_key),
(date, {_MODEL_EXPR}, api_key),
(date, {_MODEL_GROUP_EXPR}, api_key),
(date, custom_llm_provider, api_key),
(date, mcp_namespaced_tool_name, api_key),
@ -313,7 +324,7 @@ def build_key_page_sql(scope: DailyActivityScope, *, offset: int, limit: int) ->
FROM (SELECT 1) AS one
LEFT JOIN LATERAL (
SELECT * FROM ranked
ORDER BY rank_spend DESC, api_key
ORDER BY rank_spend DESC, {_API_KEY_EXPR}
LIMIT ${limit_param} OFFSET ${offset_param}
) AS page ON TRUE
"""
@ -376,7 +387,7 @@ def build_key_search_sql(scope: DailyActivityScope, *, search: str, limit: int)
WHERE {where_clause}
AND api_key <> ${sentinel_param}
AND (
api_key ILIKE ${search_param} ESCAPE '\\'
{_API_KEY_EXPR} ILIKE ${search_param} ESCAPE '\\'
OR api_key IN (
SELECT v.token FROM "LiteLLM_VerificationToken" v
LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = v.user_id
@ -392,7 +403,7 @@ def build_key_search_sql(scope: DailyActivityScope, *, search: str, limit: int)
)
)
GROUP BY api_key
ORDER BY SUM(spend::numeric) DESC, api_key
ORDER BY SUM(spend::numeric) DESC, {_API_KEY_EXPR}
LIMIT ${limit_param}
"""
return SqlQuery(sql=sql, params=(*where_params, f"%{escaped}%", PTU_SENTINEL_API_KEY, limit))
@ -406,15 +417,13 @@ def build_model_top_keys_sql(
model_param: Final = len(where_params) + 1
sentinel_param: Final = model_param + 1
limit_param: Final = sentinel_param + 1
model_clause: Final = (
f"COALESCE(NULLIF(model_group, ''), model) = ${model_param}" if by_model_group else f"model = ${model_param}"
)
model_clause: Final = f"{_MODEL_GROUP_EXPR if by_model_group else _MODEL_EXPR} = ${model_param}"
sql: Final = f"""
SELECT api_key,{_key_spend_select()}
FROM "{PRISMA_TO_PG_TABLE[scope.table]}"
WHERE {where_clause} AND {model_clause} AND api_key <> ${sentinel_param}
GROUP BY api_key
ORDER BY SUM(spend::numeric) DESC, api_key
ORDER BY SUM(spend::numeric) DESC, {_API_KEY_EXPR}
LIMIT ${limit_param}
"""
return SqlQuery(sql=sql, params=(*where_params, model_group, PTU_SENTINEL_API_KEY, limit))
@ -431,7 +440,7 @@ def build_cache_leakage_keys_sql(scope: DailyActivityScope, *, limit: int) -> Sq
WHERE {where_clause} AND api_key <> ${sentinel_param}
GROUP BY api_key
HAVING SUM(prompt_tokens) - SUM(cache_read_input_tokens) > 0
ORDER BY SUM(prompt_tokens) - SUM(cache_read_input_tokens) DESC, api_key
ORDER BY SUM(prompt_tokens) - SUM(cache_read_input_tokens) DESC, {_API_KEY_EXPR}
LIMIT ${limit_param}
"""
return SqlQuery(sql=sql, params=(*where_params, PTU_SENTINEL_API_KEY, limit))
@ -483,7 +492,7 @@ def build_export_sql(
{alias_expression} AS entity_alias,
{output_key} AS api_key,
{user_fields},
{"NULLIF(COALESCE(scoped.model, ''), '')" if export_type is ExportType.DAILY_WITH_MODELS else "NULL::text"} AS model,
{_MODEL_EXPR if export_type is ExportType.DAILY_WITH_MODELS else "NULL::text"} AS model,
COALESCE(SUM(scoped.spend), 0)::float AS spend,
{flat_cost} AS flat_cost,
COALESCE(SUM(scoped.prompt_tokens), 0)::bigint AS prompt_tokens,
@ -521,25 +530,26 @@ def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str,
)
if export_type is ExportType.DAILY_WITH_KEYS:
return (
"scoped.api_key",
"NULLIF(scoped.api_key, '')",
"MAX(COALESCE(vt.key_alias, dvt.key_alias)) AS key_alias, "
_API_KEY_EXPR,
_API_KEY_EXPR,
"MAX(CASE WHEN scoped.api_key = '' THEN 'Unattributed' "
"ELSE COALESCE(vt.key_alias, dvt.key_alias) END) AS key_alias, "
"MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, MAX(u.user_email) AS user_email",
(
'LEFT JOIN "LiteLLM_VerificationToken" vt ON vt.token = scoped.api_key',
"LEFT JOIN \"LiteLLM_VerificationToken\" vt ON vt.token = scoped.api_key AND scoped.api_key <> ''",
"""LEFT JOIN LATERAL (
SELECT key_alias, user_id
FROM "LiteLLM_DeletedVerificationToken"
WHERE token = scoped.api_key
ORDER BY deleted_at DESC
LIMIT 1
) dvt ON vt.token IS NULL""",
) dvt ON vt.token IS NULL AND scoped.api_key <> ''""",
'LEFT JOIN "LiteLLM_UserTable" u ON u.user_id = COALESCE(vt.user_id, dvt.user_id)',
),
)
if export_type is ExportType.DAILY_WITH_MODELS:
return (
"COALESCE(scoped.model, '')",
_MODEL_EXPR,
"NULL::text",
"NULL::text AS key_alias, NULL::text AS user_id, NULL::text AS user_email",
(),

12
litellm/rerank_api/main.py
View file

@ -19,6 +19,12 @@ from litellm.types.rerank import RerankResponse
from litellm.types.router import *
from litellm.utils import ProviderConfigManager, client, exception_type
_PROVIDERS_RESOLVING_THEIR_OWN_RERANK_BASE: Final = (
litellm.LlmProviders.DASHSCOPE,
litellm.LlmProviders.QWENCLOUD,
litellm.LlmProviders.QWEN_AI_PLATFORM,
)
####### ENVIRONMENT VARIABLES ###################
# Initialize any necessary instances or variables here
together_rerank: Final = TogetherAIRerank()
@ -543,7 +549,11 @@ def rerank(
logging_obj=litellm_logging_obj,
timeout=optional_params.timeout,
api_key=dynamic_api_key or optional_params.api_key,
api_base=dynamic_api_base or optional_params.api_base,
api_base=(
optional_params.api_base or dynamic_api_base
if _custom_llm_provider in _PROVIDERS_RESOLVING_THEIR_OWN_RERANK_BASE
else dynamic_api_base or optional_params.api_base
),
_is_async=_is_async,
headers=headers or litellm.headers or {},
client=client,

2
litellm/responses/streaming_iterator.py
View file

@ -227,7 +227,7 @@ async def drain_logging_tasks(timeout: float = 5.0) -> None:
return
_, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout)
if pending:
verbose_logger.warning("Timed out draining %d logging tasks on shutdown, dropping them", len(pending))
verbose_logger.warning("Timed out draining logging tasks on shutdown: %d tasks still alive", len(pending))
_ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType(

64
litellm/router.py
View file

@ -287,6 +287,11 @@ from litellm.router_utils.embedding_batch_size_check import (
embedding_deployment_selection,
log_rejection_before_provider_call,
)
from litellm.router_utils.endpoint_eligibility import (
eligible_deployments,
endpoint_selection,
warn_on_unenforced_endpoint_flag,
)
from litellm.router_utils.fallback_event_handlers import (
MID_STREAM_FALLBACK_CONTROLS_KEY,
AttemptedFallbackTargets,
@ -482,6 +487,7 @@ from litellm.utils import (
ProviderConfigManager,
Rules,
TextCompletionResponse,
aresolve_api_key_secret_reference,
function_setup,
get_llm_provider,
get_non_default_completion_params,
@ -492,7 +498,6 @@ from litellm.utils import (
lookup_model_info_for_registration,
model_info_with_pricing_overrides,
provider_rejectable_params,
resolve_api_key_secret_reference,
set_live_deployment_replay,
token_counter,
)
@ -1449,7 +1454,7 @@ class Router:
self.assistants_config = assistants_config
self.search_tools = search_tools or []
self.guardrail_list = guardrail_list or []
self.deployment_names: list = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
self.deployment_names: list[str] = [] # names of models under litellm_params. ex. azure/chatgpt-v-2
self.deployment_latency_map = {}
### CACHING ###
cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache
@ -6054,12 +6059,13 @@ class Router:
kwargs.setdefault("metadata", {}).update({"model_group": model})
# pick the one that is available (lowest TPM/RPM)
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
with endpoint_selection("text_completion", model=model):
deployment: Final = self.get_available_deployment(
model=model,
messages=messages,
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
)
data: Final = deployment["litellm_params"].copy()
for k, v in self.default_litellm_params.items():
@ -6915,6 +6921,7 @@ class Router:
"""
model_group: Final = cast(str, initial_kwargs.get("model")) # cast-ok: kwargs always carries the model group
retry_kwargs: Final = mid_stream_retry_kwargs(initial_kwargs)
handler: Final = retry_kwargs.get("original_generic_function")
healthy_deployments, all_deployments = await self._async_get_healthy_deployments(
model=model_group,
parent_otel_span=get_parent_otel_span_from_kwargs(retry_kwargs),
@ -6945,7 +6952,10 @@ class Router:
)
carry_superseded_attempt_usage(retry_kwargs)
try:
response = await attempt_function(**retry_kwargs)
with endpoint_selection(
getattr(handler, "__name__", None), getattr(handler, "__module__", None), model=model_group
):
response = await attempt_function(**retry_kwargs)
except Exception as retry_error: # noqa: BLE001 # the fallback chain judges the failed retry
wrapped = _stream_fallback_error_for_raised(retry_error, model_group, False, e.llm_provider)
if wrapped is None:
@ -9315,9 +9325,18 @@ class Router:
Handler for making a call to the .completion()/.embeddings()/etc. functions.
"""
model_group: Final = kwargs.get("model")
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
response = await response
generic_function: Final = _OPAQUE_VALUE_ADAPTER.validate_python(kwargs.get("original_generic_function"))
endpoint_function: Final = (
generic_function if callable(generic_function) else _OPAQUE_VALUE_ADAPTER.validate_python(original_function)
)
with endpoint_selection(
getattr(endpoint_function, "__name__", None),
getattr(endpoint_function, "__module__", None),
model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group),
):
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
response = await response
await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs)
## PROCESS RESPONSE HEADERS
response = await self.set_response_headers(response=response, model_group=model_group, request_kwargs=kwargs)
@ -10638,6 +10657,7 @@ class Router:
declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params)
warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params)
warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params)
warn_on_unenforced_endpoint_flag(model_name=_model_name, model_info=_model_info)
deployment = without_model_capability_flags(
Deployment(
**deployment_info,
@ -11441,6 +11461,9 @@ class Router:
declared_params: Final = deployment.litellm_params.model_dump(exclude_none=True)
warn_on_provider_credential_mismatch(model_name=deployment.model_name, litellm_params=declared_params)
warn_on_ignored_model_capability_flags(model_name=deployment.model_name, litellm_params=declared_params)
warn_on_unenforced_endpoint_flag(
model_name=deployment.model_name, model_info=deployment.model_info.model_dump()
)
served: Final = with_admin_api_key_secret_reference(without_model_capability_flags(deployment))
# add to model list
@ -12056,7 +12079,7 @@ class Router:
return
if api_base is None or "*" in model or params.get("use_clientside_credentials"):
return
api_key: Final = resolve_api_key_secret_reference(
api_key: Final = await aresolve_api_key_secret_reference(
params.api_key or dynamic_api_key,
model=model,
llm_provider=provider,
@ -14797,9 +14820,15 @@ class Router:
"""
return [m for m in self.model_list if m["litellm_params"]["model"] == model]
def get_routable_deployments(self, model: str) -> tuple[DeploymentTypedDict, ...]:
def get_routable_deployments(
self,
model: str,
request_kwargs: Mapping[str, object] = MappingProxyType({}),
call_type: str | None = None,
) -> tuple[DeploymentTypedDict, ...]:
try:
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs={}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
with endpoint_selection(call_type, model=model):
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
except (litellm.BadRequestError, ValueError):
return ()
if isinstance(candidates, dict):
@ -15075,7 +15104,10 @@ class Router:
model=model,
llm_provider="",
)
return selectable
eligible: Final = eligible_deployments(model, selectable)
if isinstance(eligible, tuple):
return list(eligible)
raise eligible
def _filter_reserved_deployments(
self,

120
litellm/router_utils/endpoint_eligibility.py Normal file
View file

@ -0,0 +1,120 @@
from collections.abc import Generator, Mapping, Sequence
from contextlib import contextmanager
from contextvars import ContextVar
from types import MappingProxyType
from typing import Final, TypeVar
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, field_validator
from litellm._logging import verbose_router_logger
from litellm.exceptions import EndpointNotSupportedByModelError
CHAT_ENDPOINT_BY_CALL: Final[Mapping[str, str]] = MappingProxyType(
{
"completion": "/v1/chat/completions",
"acompletion": "/v1/chat/completions",
"_completion": "/v1/chat/completions",
"_acompletion": "/v1/chat/completions",
"text_completion": "/v1/completions",
"atext_completion": "/v1/completions",
"_atext_completion": "/v1/completions",
"anthropic_messages": "/v1/messages",
"aresponses": "/v1/responses",
"agenerate_content": "generateContent",
"agenerate_content_stream": "streamGenerateContent",
"acreate_interaction": "/v1beta/interactions",
"litellm.interactions.main.acreate": "/v1beta/interactions",
"litellm.interactions.main.create": "/v1beta/interactions",
"_aresponses_websocket": "/v1/responses",
"acompact_responses": "/v1/responses/compact",
}
)
def _endpoint_path(endpoint: str) -> str:
rooted: Final = "/" + endpoint.strip().strip("/").casefold()
path: Final = rooted.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1")
return "/" + path.rsplit(":", 1)[-1] if ":" in path else path
CHAT_ENDPOINTS: Final = frozenset(_endpoint_path(endpoint) for endpoint in CHAT_ENDPOINT_BY_CALL.values())
_Deployment: Final = TypeVar("_Deployment", bound=Mapping[str, object])
_ENFORCE_FLAG: Final = TypeAdapter(bool | None)
_REQUEST_SELECTION: Final[ContextVar[tuple[str | None, str | None]]] = ContextVar(
"router_request_endpoint", default=(None, None)
)
class _EndpointDeclaration(BaseModel):
model_config = ConfigDict(frozen=True)
enforce_supported_endpoints: bool | None = None
supported_endpoints: tuple[str, ...] | None = None
@field_validator("supported_endpoints", mode="before")
@classmethod
def _one_endpoint_as_a_list(cls, value: object) -> object:
return (value,) if isinstance(value, str) else value
@contextmanager
def endpoint_selection(call_type: object, module: object = None, model: object = None) -> Generator[None]:
qualified_call: Final = f"{module}.{call_type}" if module == "litellm.interactions.main" else call_type
endpoint: Final = CHAT_ENDPOINT_BY_CALL.get(qualified_call) if isinstance(qualified_call, str) else None
token: Final = _REQUEST_SELECTION.set((endpoint, model if isinstance(model, str) else None))
try:
yield
finally:
_REQUEST_SELECTION.reset(token)
def _enforced_endpoints(model_info: object) -> tuple[str, ...] | None:
try:
declaration: Final = _EndpointDeclaration.model_validate(model_info)
except ValidationError:
return None
return declaration.supported_endpoints if declaration.enforce_supported_endpoints else None
def warn_on_unenforced_endpoint_flag(model_name: str, model_info: Mapping[str, object]) -> str | None:
try:
flagged: Final = _ENFORCE_FLAG.validate_python(model_info.get("enforce_supported_endpoints"))
except ValidationError:
return None
if not flagged or _enforced_endpoints(model_info) is not None:
return None
warning: Final = (
f"Deployment '{model_name}' sets enforce_supported_endpoints without a supported_endpoints list of "
'strings, so litellm enforces nothing for it. Set supported_endpoints, e.g. ["/v1/embeddings"].'
)
verbose_router_logger.warning(warning)
return warning
def non_chat_endpoints(deployment: Mapping[str, object]) -> frozenset[str] | None:
declared: Final = _enforced_endpoints(deployment.get("model_info"))
if declared is None:
return None
endpoints: Final = frozenset(declared)
normalized: Final = frozenset(_endpoint_path(endpoint) for endpoint in endpoints)
return endpoints if normalized.isdisjoint(CHAT_ENDPOINTS) else None
def eligible_deployments(
model: str, deployments: Sequence[_Deployment]
) -> tuple[_Deployment, ...] | EndpointNotSupportedByModelError:
endpoint, requested_model = _REQUEST_SELECTION.get()
if endpoint is None or not deployments:
return tuple(deployments)
eligible: Final = tuple(deployment for deployment in deployments if non_chat_endpoints(deployment) is None)
if eligible:
return eligible
requested: Final = requested_model or model
supported: Final = ", ".join(
sorted(frozenset[str]().union(*(non_chat_endpoints(deployment) or frozenset() for deployment in deployments)))
)
return EndpointNotSupportedByModelError(
message=f"{endpoint} is not supported by model={requested}. Supported endpoints: {supported or 'none'}",
model=model,
llm_provider="",
)

2
litellm/rust_bridge/_native.pyi
View file

@ -14,6 +14,7 @@ class ForkedAfterNativeRuntimeStarted(RuntimeError): ...
class ProcessReservedForForking(RuntimeError): ...
def trace_encode_error(message: str) -> bytes: ...
def bundled_model_cost_map() -> bytes: ...
@final
class NativeClickHouseSpendConfig:
@ -220,6 +221,7 @@ __all__ = [
"aocr",
"aresponses",
"atranscription",
"bundled_model_cost_map",
"completion",
"embedding",
"gil_stats",

135
litellm/secret_managers/main.py
View file

@ -5,7 +5,7 @@ import time
import traceback
from collections.abc import Callable
from dataclasses import dataclass
from typing import Final
from typing import Final, TypeAlias
import httpx
from pydantic import TypeAdapter, ValidationError
@ -13,6 +13,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching import DualCache
from litellm.litellm_core_utils.env_utils import get_env_float
from litellm.litellm_core_utils.single_flight import single_flight_lru_cache
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.secret_managers.get_azure_ad_token_provider import (
@ -167,12 +168,37 @@ def get_secret_str(
LOCAL_OIDC_SECRET_PREFIXES: Final = ("oidc/file/", "oidc/env/")
GOOGLE_OIDC_SECRET_PREFIX: Final = "oidc/google/"
API_KEY_OIDC_SECRET_PREFIXES: Final = (*LOCAL_OIDC_SECRET_PREFIXES, GOOGLE_OIDC_SECRET_PREFIX)
_GOOGLE_METADATA_DEFAULT_HOST: Final = "metadata.google.internal"
_GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS: Final = 3.0
_GOOGLE_TOKEN_UNAVAILABLE: Final = "the metadata server did not return an ID token"
def _google_metadata_identity_url() -> str:
host: Final = os.getenv("GCE_METADATA_HOST") or _GOOGLE_METADATA_DEFAULT_HOST
return f"http://{host}/computeMetadata/v1/instance/service-accounts/default/identity"
def _google_metadata_timeout() -> float:
seconds: Final = get_env_float("GCE_METADATA_TIMEOUT", _GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS)
return seconds if seconds > 0 else _GOOGLE_METADATA_DEFAULT_TIMEOUT_SECONDS
class GoogleMetadataStatusError(ValueError):
def __init__(self, status_code: int) -> None:
super().__init__(f"Google OIDC provider failed with HTTP {status_code}")
self.status_code: Final = status_code
@dataclass(frozen=True, slots=True)
class UnresolvableSecretReference:
kind: str
reason: str
retryable: bool = False
OidcResolver: TypeAlias = Callable[[str], "str | UnresolvableSecretReference"]
_OIDC_PROVIDERS: Final = frozenset({"google", "circleci", "circleci_v2", "github", "azure", "file", "env", "env_path"})
@ -198,11 +224,86 @@ def resolve_local_oidc_reference(value: str) -> str | UnresolvableSecretReferenc
return secret
def _logged_unresolvable(reference: str, kind: str, reason: str) -> UnresolvableSecretReference:
def resolve_api_key_oidc_reference(value: str) -> str | UnresolvableSecretReference:
if not value.startswith(GOOGLE_OIDC_SECRET_PREFIX):
return resolve_local_oidc_reference(value)
try:
token: Final = _google_id_token(
value, value.removeprefix(GOOGLE_OIDC_SECRET_PREFIX), timeout=_google_metadata_timeout()
)
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as exc:
return _logged_unresolvable(
value,
"oidc/google",
_GOOGLE_TOKEN_UNAVAILABLE,
f"{type(exc).__name__}: {exc}",
retryable=_metadata_server_failed_to_answer(exc),
)
if not token:
return _logged_unresolvable(value, "oidc/google", "it resolved to an empty value")
return token
def _metadata_server_failed_to_answer(exc: httpx.HTTPError | httpx.InvalidURL | ValueError) -> bool:
match exc:
case httpx.TransportError():
return True
case GoogleMetadataStatusError(status_code=status_code):
return status_code >= 500
case _:
return False
def _google_id_token(reference: str, audience: str, timeout: float | None) -> str:
cached: Final = cached_oidc_token(reference)
if cached is not None:
return cached
oidc_client: Final = _get_oidc_http_handler()
try:
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
response: Final = oidc_client.get(
_google_metadata_identity_url(),
params={"audience": audience},
headers={"Metadata-Flavor": "Google"},
timeout=timeout,
)
finally:
oidc_client.close()
if response.status_code != 200:
raise GoogleMetadataStatusError(response.status_code)
token: Final = response.text
ttl: Final = _oidc_token_cache_ttl(token, 3600 - 60)
if not token:
verbose_logger.warning("Google OIDC token for %s is empty; not caching it", _single_line(reference))
elif ttl > 0:
oidc_cache.set_cache(key=reference, value=token, ttl=ttl)
else:
verbose_logger.warning(
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
_single_line(reference),
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
)
return token
def cached_oidc_token(reference: str) -> str | None:
cached: Final = oidc_cache.get_cache(key=reference) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # DualCache.get_cache is untyped
return cached if isinstance(cached, str) and cached else None
def _logged_unresolvable(
reference: str, kind: str, reason: str, detail: str | None = None, retryable: bool = False
) -> UnresolvableSecretReference:
verbose_logger.warning(
"Secret reference %s could not be resolved: %s", reference.replace("\r", "").replace("\n", ""), reason
"Secret reference %s could not be resolved: %s",
_single_line(reference),
reason if detail is None else f"{reason} ({_single_line(detail)})",
)
return UnresolvableSecretReference(kind=kind, reason=reason)
return UnresolvableSecretReference(kind=kind, reason=reason, retryable=retryable)
def _single_line(text: str) -> str:
return text.replace("\r", "").replace("\n", "")
def _unresolvable_reason(kind: str, exc: OSError | ValueError) -> str:
@ -273,31 +374,7 @@ def get_secret(
oidc_aud = "/".join(secret_name_split.split("/")[1:])
# TODO: Add caching for HTTP requests
if oidc_provider == "google":
oidc_token = oidc_cache.get_cache(key=secret_name)
if oidc_token is not None:
return oidc_token
oidc_client = _get_oidc_http_handler()
# https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature
response = oidc_client.get(
"http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity",
params={"audience": oidc_aud},
headers={"Metadata-Flavor": "Google"},
)
if response.status_code == 200:
oidc_token = response.text
ttl: Final = _oidc_token_cache_ttl(oidc_token, 3600 - 60)
if ttl > 0:
oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=ttl)
else:
verbose_logger.warning(
"Google OIDC token for %s is already expired or expires within %ss; not caching it",
secret_name,
_OIDC_TOKEN_EXPIRY_MARGIN_SECONDS,
)
return oidc_token
else:
raise ValueError("Google OIDC provider failed")
return _google_id_token(secret_name, oidc_aud, timeout=None)
elif oidc_provider == "circleci":
# https://circleci.com/docs/openid-connect-tokens/
env_secret = os.getenv("CIRCLE_OIDC_TOKEN")

1
litellm/types/router.py
View file

@ -270,6 +270,7 @@ def _whole_float_as_int(value: object) -> object:
class ModelInfo(MirroredPricingParams):
enforce_supported_endpoints: bool | None = None
id: str | None # Allow id to be optional on input, but it will always be present as a str in the model instance
db_model: bool = False # used for proxy - to separate models which are stored in the db vs. config.
updated_at: datetime.datetime | None = None

8
litellm/types/utils.py
View file

@ -4147,7 +4147,12 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
{"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"}
{
"supports_regex_lookaround",
"supports_assistant_prefill",
"supports_bedrock_native_structured_output",
"supports_pdf_input",
}
)
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})
@ -4559,6 +4564,7 @@ class LlmProviders(str, Enum):
S3_VECTORS = "s3_vectors"
VALKEY = "valkey"
MONGODB = "mongodb"
OPENSEARCH = "opensearch"
HELICONE = "helicone"
HYPERBOLIC = "hyperbolic"
RECRAFT = "recraft"

Some files were not shown because too many files have changed in this diff Show more