scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-24 05:45:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 72

Merge remote-tracking branch 'origin/main' into fix45874

Browse source
This commit is contained in:
yucheng-berri 2026-10-17 01:01:41 -07:00
commit 40903e3f7a
688 changed files with 68891 additions and 7454 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
.github/scripts/assert_ci_coverage.py vendored
View file

@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
# tests has to be named by some shard or it runs nowhere. A child listed here is
@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
return frozenset(ast.literal_eval(values))
def _owns(workflow: str, relative_path: str) -> bool:
owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
return owned_root is not None and relative_path.startswith(owned_root)
def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
return frozenset[str]().union(
*(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
)
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
runner: Final = repo_root / "tests/integration/run.py"
if not runner.exists():
@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
)
!= 1
)
gha_tokens: Final = _invoked_test_tokens(
scalar
for path in (repo_root / ".github/workflows").glob("*.y*ml")
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
workflow_tokens: Final = MappingProxyType(
{
path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
for path in (repo_root / ".github/workflows").glob("*.y*ml")
}
)
gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
circle_owned: Final = (
frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
)
findings: Final = (
tuple(
Finding(path, "integration contract is also selected by GitHub Actions")
for path in paths
Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
for path in sorted(circle_owned)
if any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
for path in sorted(github_files)
if not (repo_root / path).is_file()
)
+ tuple(
Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
for path in sorted(github_files)
if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
)
)
browser_commands: Final = tuple(
scalar.value
@ -688,13 +710,9 @@ def main() -> int:
scalars = _all_scalars()
integration_paths, ownership_findings = _integration_ownership()
test_findings = (
_uncovered_tests(
allowlist,
_invoked_selections(scalars)
+ (Selection(included=integration_paths, ignored=frozenset()),),
)
+ ownership_findings
test_findings = _uncovered_tests(
allowlist,
_invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
)
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
@ -711,13 +729,25 @@ def main() -> int:
test_findings,
"Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if ownership_findings:
browser_remedy = (
" CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
"select them and the default Playwright config must exclude them."
if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
else ""
)
_report(
"integration contracts with the wrong CI owner",
ownership_findings,
"Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
)
if dockerfile_findings:
_report(
"Dockerfiles that no CI job builds",
dockerfile_findings,
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if stale_findings or test_findings or dockerfile_findings:
if stale_findings or test_findings or ownership_findings or dockerfile_findings:
return 1
_write(

5
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -24,12 +24,9 @@ jobs:
- name: Update JSON Data
run: |
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
- name: Regenerate JSON Schema
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create Pull Request
run: |
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
git add model_prices_and_context_window.json
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
gh pr create --title "Update model_prices_and_context_window.json file" \
--body "Automated update for model_prices_and_context_window.json" \

7
.github/workflows/sync-together-ai-models.yml vendored
View file

@ -41,10 +41,6 @@ jobs:
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Regenerate the JSON schema
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
@ -57,8 +53,7 @@ jobs:
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json \
model_prices_and_context_window.schema.json
litellm/model_prices_and_context_window_backup.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"

8
.github/workflows/test-rust.yml vendored
View file

@ -16,6 +16,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"
@ -46,6 +50,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"

1
.github/workflows/test-unit.yml vendored
View file

@ -696,7 +696,6 @@ jobs:
tests/unit/responses/litellm_completion_transformation
tests/unit/responses/test_additional_tools.py
tests/unit/responses/test_custom_tool_call.py
tests/unit/responses/test_dispatch.py
tests/unit/responses/test_metadata_codex_callback.py
tests/unit/responses/test_no_duplicate_spend_logs.py
tests/unit/responses/test_null_test_fix.py

4
AGENTS.md
View file

@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

1
README.md
View file

@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nace (`nace`)](https://console.nace.ai/docs/drex) | | | | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |

32
backend/Dockerfile
View file

@ -4,6 +4,33 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
FROM $UV_IMAGE AS uvbin
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -54,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -63,6 +90,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

45
ci_cd/cost_map_guard.py
View file

@ -1,11 +1,11 @@
"""Guard the cost map on pull requests.
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
files and may only add or update models.
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files or the legacy schema
path and may only add or update models.
"""
from __future__ import annotations
@ -19,13 +19,12 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
CostMap = dict[str, object]
@ -35,7 +34,6 @@ CostMap = dict[str, object]
class Snapshot:
cost_map: str
backup: str
schema: str
def _parse_object(text: str, path: str) -> CostMap | str:
@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str:
return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
def _rendered_schema(cost_map: CostMap) -> str:
try:
return render(build_schema(cost_map))
except SystemExit as error:
return str(error)
def _evaluation_output_price_failure(
key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
) -> str | None:
@ -78,29 +69,13 @@ def _file_failures(
*,
evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
) -> tuple[str, ...]:
schema_text: Final = _rendered_schema(head_map)
if not schema_text.startswith("{"):
return (schema_text,)
backup_failure: Final = (
()
if head.backup == head.cost_map
else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
)
schema_failure: Final = (
()
if head.schema == schema_text
else (
f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
"run `python ci_cd/generate_model_prices_schema.py` and commit the result",
)
)
return (
*backup_failure,
*schema_failure,
*(
f"{COST_MAP_PATH} does not validate against its schema: {error}"
for error in validation_errors(head_map, json.loads(schema_text))[:20]
),
*(
failure
for key, entry in head_map.items()
@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
*(
f"bot PRs may only change the cost map files, not {path}"
for path in changed_files
if path not in GUARDED_PATHS
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
),
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

418
ci_cd/generate_model_prices_schema.py
View file

@ -1,418 +0,0 @@
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Final, Optional
import jsonschema
REPO_ROOT = Path(__file__).parent.parent
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
JsonSchema = dict
NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
BOOLEAN: JsonSchema = {"type": "boolean"}
STRING: JsonSchema = {"type": "string"}
TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
WEEKDAY_PATTERN: Final = (
r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
)
EXTRA_BOOLEAN_KEYS = frozenset(
{
"gemini_native_audio",
"gemini_audio_only_live",
"uses_embed_content",
"use_openai_responses_path",
"bedrock_converse_supports_strict_tools",
"thinking_always_on",
}
)
HOURS_UTC: Final[JsonSchema] = {
"description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.',
"oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}],
}
OFF_PEAK_WINDOW: Final[JsonSchema] = {
"type": "object",
"properties": {
"hours_utc": HOURS_UTC,
"weekdays": {
"type": "array",
"description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.",
"items": {
"oneOf": [
{"type": "integer", "minimum": 1, "maximum": 7},
{"type": "string", "pattern": WEEKDAY_PATTERN},
]
},
"minItems": 1,
},
},
"required": ["hours_utc"],
"additionalProperties": False,
}
OBJECT_KEYS: dict[str, JsonSchema] = {
"off_peak_pricing": {
"type": "object",
"description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
"properties": {
"hours_utc": HOURS_UTC,
"windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1},
"weekday_timezone": {
"type": "string",
"description": "IANA zone the weekdays of each window are read on; defaults to UTC.",
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
},
"anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}],
"additionalProperties": False,
},
"search_context_cost_per_query": {
"type": "object",
"description": "USD cost per web search query, keyed by search context size.",
"properties": {
"search_context_size_low": NONNEG_NUMBER,
"search_context_size_medium": NONNEG_NUMBER,
"search_context_size_high": NONNEG_NUMBER,
},
"additionalProperties": False,
},
"guardrail_cost_per_unit": {
"type": "object",
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
"additionalProperties": NONNEG_NUMBER,
},
"metadata": {
"type": "object",
"description": "Free-form notes about the entry (e.g. pricing derivation).",
},
"provider_specific_entry": {
"type": "object",
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
},
}
ARRAY_KEYS: dict[str, JsonSchema] = {
"supported_audio_formats": {
"type": "array",
"description": "Audio container formats the model can return.",
"items": {"type": "string", "enum": ["mp3", "wav"]},
},
"supported_endpoints": {
"type": "array",
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
"items": STRING,
},
"supported_modalities": {
"type": "array",
"description": "Input modalities the model accepts.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
},
"supported_output_modalities": {
"type": "array",
"description": "Output modalities the model can produce.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
},
"reasoning_effort_levels": {
"type": "array",
"description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
"items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
},
"supported_regions": {
"type": "array",
"description": "Cloud regions the model is available in ('global' or region ids).",
"items": STRING,
},
"tiered_pricing": {
"type": "array",
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
"items": {
"type": "object",
"properties": {
"range": {
"type": "array",
"description": "[min, max] prompt-token span this tier applies to.",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"max_results_range": {
"type": "array",
"description": "[min, max] result-count span this tier applies to (search models).",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
"input_cost_per_query": NONNEG_NUMBER,
},
"additionalProperties": False,
},
},
}
INTEGER_KEYS: dict[str, JsonSchema] = {
"max_tokens": {
**NONNEG_INTEGER,
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
},
"max_input_tokens": {
**NONNEG_INTEGER,
"description": "Maximum prompt/context tokens the model accepts.",
},
"max_output_tokens": {
**NONNEG_INTEGER,
"description": "Maximum tokens the model can generate in one response.",
},
"output_vector_size": {
**NONNEG_INTEGER,
"description": "Embedding dimension for embedding models.",
},
"prompt_cache_min_tokens": {
**NONNEG_INTEGER,
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
},
"token_count_framing": {
**NONNEG_INTEGER,
"description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
},
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
}
NUMBER_KEYS: dict[str, JsonSchema] = {
"regional_processing_uplift_multiplier_eu": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
},
"regional_processing_uplift_multiplier_us": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
},
"regional_endpoint_uplift_multiplier": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
},
}
COST_DESCRIPTIONS: dict[str, str] = {
"input_cost_per_token": "USD per prompt token.",
"output_cost_per_token": "USD per generated token.",
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
"google_maps_grounding_cost_per_query": (
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
}
def cost_description(key: str) -> Optional[str]:
if key in COST_DESCRIPTIONS:
return COST_DESCRIPTIONS[key]
if key.endswith("_flex"):
return "Flex service-tier rate for the same-named base field."
if key.endswith("_priority"):
return "Priority service-tier rate for the same-named base field."
if "_above_" in key:
return "Rate applied once the prompt exceeds the token threshold in the field name."
return None
def cost_schema(key: str) -> JsonSchema:
description = cost_description(key)
return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
return {
"litellm_provider": {
"type": "string",
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
},
"mode": {
"type": "string",
"description": "Primary API surface / task type of the model.",
"enum": list(modes),
},
"source": {
"type": "string",
"description": "URL of the provider pricing/model page this entry was taken from.",
},
"deprecation_date": {
"type": "string",
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
"format": "date",
"pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
},
"web_search_billing_unit": {
"type": "string",
"description": "Whether web search is billed per query or per prompt.",
"enum": ["per_query", "per_prompt"],
},
"bedrock_output_config_effort_ceiling": {
"type": "string",
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"token_count_fallback_model": {
"type": "string",
"minLength": 1,
"description": "Model that counts tokens for this one when the provider cannot count it.",
},
"audio_transcription_config": STRING,
"vertex_ai_audio_api": {
"type": "string",
"enum": ["lyria_predict", "lyria_interactions"],
},
}
def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
if key in curated:
return curated[key]
if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
return BOOLEAN
if "cost" in key:
return cost_schema(key)
return None
def build_schema(prices: dict) -> JsonSchema:
entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
if unclassified:
raise SystemExit(
f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
f"Add them to the key tables in {Path(__file__).name} and rerun it."
)
entry_properties = {key: classify(key, modes) for key in all_keys}
return {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "LiteLLM model_prices_and_context_window.json",
"description": (
"Schema for LiteLLM's model price and context window registry "
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
"ignore unknown fields rather than reject them."
),
"type": "object",
"properties": {
"sample_spec": {
"type": "object",
"description": (
"Documentation placeholder illustrating the entry shape; not a real model and not "
"schema-conformant (several values are prose)."
),
},
"fallback_generalizations": {
"type": "object",
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
"properties": {
"rules": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": STRING,
"pattern": STRING,
"description": STRING,
},
"required": ["name", "pattern"],
"additionalProperties": True,
},
}
},
"additionalProperties": False,
},
},
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
"$defs": {
"modelEntry": {
"type": "object",
"description": (
"Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
"are optional; boolean capability flags are simply omitted when unknown or false."
),
"required": ["litellm_provider"],
"properties": entry_properties,
"additionalProperties": True,
}
},
}
def render(schema: JsonSchema) -> str:
return json.dumps(schema, indent=2) + "\n"
def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
return tuple(
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
for error in validator.iter_errors(prices)
)
def main() -> int:
check = "--check" in sys.argv[1:]
prices = json.loads(PRICES_PATH.read_text())
rendered = render(build_schema(prices))
errors = validation_errors(prices, json.loads(rendered))
if errors:
print(f"{PRICES_PATH.name} does not validate against the generated schema:")
print("\n".join(errors[:20]))
return 1
if not check:
SCHEMA_PATH.write_text(rendered)
print(f"wrote {SCHEMA_PATH}")
return 0
if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
print(
f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
)
return 1
print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
return 0
if __name__ == "__main__":
sys.exit(main())

34
gateway/Dockerfile
View file

@ -20,6 +20,33 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -66,11 +93,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
# Stage 2 — copy source and install the workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -79,6 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20261015120000_add_environment_variables_revision_trigger/migration.sql Normal file
View file

@ -0,0 +1,23 @@
SET LOCAL lock_timeout = '10s';
CREATE SEQUENCE IF NOT EXISTS "litellm_environment_variables_revision_seq";
GRANT USAGE ON SEQUENCE "litellm_environment_variables_revision_seq" TO PUBLIC;
CREATE OR REPLACE FUNCTION litellm_bump_environment_variables_revision() RETURNS TRIGGER AS $$
DECLARE
next_revision BIGINT := nextval(format('%I.litellm_environment_variables_revision_seq', TG_TABLE_SCHEMA)::regclass);
BEGIN
IF TG_OP = 'UPDATE' THEN
NEW."reload_revision" := GREATEST(next_revision, OLD."reload_revision" + 1);
ELSE
NEW."reload_revision" := next_revision;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS "litellm_environment_variables_revision" ON "LiteLLM_Config";
CREATE TRIGGER "litellm_environment_variables_revision"
BEFORE INSERT OR UPDATE ON "LiteLLM_Config"
FOR EACH ROW WHEN (NEW."param_name" = 'environment_variables')
EXECUTE PROCEDURE litellm_bump_environment_variables_revision();

2
litellm-proxy-extras/litellm_proxy_extras/migrations/20261016000000_background_interaction_settlement_settled_at_index/migration.sql Normal file
View file

@ -0,0 +1,2 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_background_interaction_settlement_settled_at"
ON "LiteLLM_BackgroundInteractionSettlement" ("settled_at");

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1981,6 +1981,7 @@ model LiteLLM_BackgroundInteractionSettlement {
outcome String?
@@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
@@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
}
model LiteLLM_Lens {

2
litellm-rust/crates/gateway-inference/src/messages.rs
View file

@ -24,7 +24,7 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
pub async fn create(
State(gateway): State<Arc<Gateway>>,

2
litellm-rust/crates/gateway-inference/tests/messages.rs
View file

@ -16,7 +16,7 @@ use wiremock::{
#[rstest]
#[case::anthropic("anthropic/test-model", "/v1/messages", true)]
#[case::azure("azure_ai/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", true)]
#[tokio::test]
async fn messages_reaches_the_provider_and_preserves_json_or_sse(
#[case] model: &str,

1
litellm-rust/crates/model-catalog/AGENTS.md
View file

@ -2,5 +2,4 @@
For `model_prices_and_context_window.json` validation, we should eventually:
- Remove any schema file like `model_prices_and_context_window.schema.json`
- Stop skipping this crate's tests

38
litellm/__init__.py
View file

@ -1,34 +1,13 @@
### Hide pydantic namespace conflict warnings globally ###
from __future__ import annotations
import warnings
from litellm._process_defaults import apply_process_defaults as _apply_process_defaults
_apply_process_defaults()
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
### INIT VARIABLES #########################
import threading
import os
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
_dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
from collections.abc import Mapping, Sequence
from typing import (
Any,
@ -207,7 +186,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]]
[]
) # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
post_call_rules: List[Callable[..., object]] = []
turn_off_message_logging: Optional[bool] = False
standard_logging_payload_excluded_fields: Optional[List[str]] = (
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
@ -393,6 +372,7 @@ key_alias_pattern: str | None = None
enable_gemini_default_thinking_level_low: bool = (
False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
)
gemini_include_thoughts_default: bool = True
####################
logging: bool = True
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
@ -1176,7 +1156,9 @@ def _build_models_by_provider() -> dict:
"edenai": edenai_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
| vertex_code_chat_models
| vertex_text_models
| vertex_code_text_models
| vertex_anthropic_models
| vertex_vision_models
| vertex_language_models
@ -1184,7 +1166,7 @@ def _build_models_by_provider() -> dict:
| vertex_minimax_models
| vertex_moonshot_models
| vertex_zai_models,
"ai21": ai21_models,
"ai21": ai21_models | ai21_chat_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
@ -1415,6 +1397,7 @@ from .exceptions import (
InvalidRequestError,
BadRequestError,
ImageFetchError,
MalformedRequestFieldError as MalformedRequestFieldError,
VectorStoreSearchError,
NotFoundError,
PermissionDeniedError,
@ -1690,6 +1673,9 @@ if TYPE_CHECKING:
from .llms.perplexity.decisions.transformation import (
PerplexityDecisionsConfig as PerplexityDecisionsConfig,
)
from .llms.nace.decisions.transformation import (
NaceDecisionsConfig as NaceDecisionsConfig,
)
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)

10
litellm/_internal_context.py
View file

@ -37,6 +37,16 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
@contextmanager
def internal_sub_call() -> Generator[None]:
"""A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
token: Final = is_internal_call.set(True)
try:
yield
finally:
is_internal_call.reset(token)
@contextmanager
def post_response_phase() -> Generator[None]:
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

2
litellm/_lazy_imports_registry.py
View file

@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"HostedVLLMRerankConfig",
),
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),

24
litellm/_process_defaults.py Normal file
View file

@ -0,0 +1,24 @@
import os
import warnings
import dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
def apply_process_defaults() -> None:
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())

8
litellm/assistants/main.py
View file

@ -129,6 +129,7 @@ def get_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_assistants=aget_assistants,
)
@ -312,6 +313,7 @@ def create_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
create_assistant_data=create_assistant_data,
client=client,
async_create_assistants=async_create_assistants,
@ -461,6 +463,7 @@ def delete_assistant(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
assistant_id=assistant_id,
client=client,
async_delete_assistants=async_delete_assistants,
@ -637,6 +640,7 @@ def create_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
acreate_thread=acreate_thread,
)
@ -783,6 +787,7 @@ def get_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_thread=aget_thread,
)
@ -960,6 +965,7 @@ def add_message(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
a_add_message=a_add_message,
)
@ -1111,6 +1117,7 @@ def get_messages(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_messages=aget_messages,
)
@ -1308,6 +1315,7 @@ def run_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
arun_thread=arun_thread,
event_handler=event_handler,

3
litellm/batches/batch_utils.py
View file

@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
)
from litellm.types.llms.openai import Batch
from litellm.types.utils import ModelInfo, Usage
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
from litellm.utils import token_counter
@ -569,6 +569,7 @@ def extract_file_access_credentials(
# A federated deployment holds no api_key, so without these the fetch that reads a
# finished batch's output has nothing to authenticate with and its cost is never billed.
*sorted(ANTHROPIC_WIF_KWARGS_KEYS),
*sorted(OPENAI_WIF_KWARGS_KEYS),
)
for key in credential_keys:
if key in litellm_params:

4
litellm/batches/main.py
View file

@ -309,6 +309,7 @@ def create_batch(
timeout=timeout,
max_retries=optional_params.max_retries,
_is_async=_is_async,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -819,6 +821,7 @@ def list_batches(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -1016,6 +1019,7 @@ def cancel_batch(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

13
litellm/caching/caching.py
View file

@ -25,7 +25,7 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
from litellm.types.caching import *
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
@ -376,7 +376,7 @@ class Cache:
)
return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
def get_cache_key(self, **kwargs) -> str:
def get_cache_key(self, **kwargs: object) -> str:
"""
Get the cache key for the given arguments.
@ -410,13 +410,16 @@ class Cache:
if litellm.enable_caching_on_provider_specific_optional_params is True: # feature flagged for now
if kwargs[param] is None:
continue # ignore None params
param_value = kwargs[param]
cache_key += f"{param}: {param_value}"
cache_key += f"{param}: {kwargs[param]}"
if is_semantic_cache:
cache_key += self._get_semantic_cache_tenant_scope(kwargs)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
reasoning_summary: Final = peek_reasoning_summary_aliases(kwargs)
reasoning_summary_material: Final = (
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",

13
litellm/caching/caching_handler.py
View file

@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
import litellm
from litellm._internal_context import post_response_phase
from litellm._internal_context import is_internal_call, post_response_phase
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache, response_cache_phase
@ -511,8 +511,10 @@ class LLMCachingHandler:
custom_llm_provider=custom_llm_provider,
)
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
logging_obj.handle_sync_success_callbacks_for_async_calls(
if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
cached_result=cached_result
):
logging_obj.handle_sync_cache_hit_success_callbacks(
result=cached_result,
start_time=start_time,
end_time=end_time,
@ -820,7 +822,12 @@ class LLMCachingHandler:
start_time (datetime): The start time of the operation.
end_time (datetime): The end time of the operation.
cache_hit (bool): Whether it was a cache hit.
An internal sub-call's hit fires nothing, since the parent call that folds it in logs it.
"""
if is_internal_call.get():
return
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

6
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -197,7 +197,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
return not _supports_audio_input(model, litellm_params)
def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
return MappingProxyType(
{
key: value
@ -1455,7 +1455,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
for key, value in _provider_metadata(raw_response.model_extra).items():
for key, value in responses_provider_metadata(raw_response.model_extra).items():
setattr(model_response, key, value)
# Preserve hidden params from the ResponsesAPIResponse, especially the headers
@ -2139,7 +2139,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
from litellm.responses.utils import ResponseAPILoggingUtils
usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
provider_metadata: Final = _provider_metadata(response_data)
provider_metadata: Final = responses_provider_metadata(response_data)
served_service_tier: Final = response_data.get("service_tier")
return ModelResponseStream(
choices=[

12
litellm/constants.py
View file

@ -1,4 +1,5 @@
import os
import re
import sys
from enum import Enum
from types import MappingProxyType
@ -18,6 +19,9 @@ MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_PROFILE: Final = "jwt_bearer_obo"
MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_SCOPE: Final = "https://graph.microsoft.com/.default"
# Replies depend on the caller's delegated identity, which response-cache keys do not include.
RESPONSE_CACHE_EXCLUDED_PROVIDERS: Final = frozenset({"microsoft_365_copilot", "databricks_agent"})
CLAUDE_CODE_PICKER_PATTERN: Final = re.compile(r"claude|anthropic", re.IGNORECASE)
GATEWAY_CLIENT_HEADER: Final = "x-gateway-client"
CLAUDE_CODE_CLIENT: Final = "claude-code"
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
@ -57,6 +61,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_mid_stream_fallback_continuation",
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",
@ -146,6 +151,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
# Streams exceeding this duration are terminated with a Timeout error.
# None (default) = no limit. Set env var to a number of seconds to enable globally.
LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS")
MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Maximum number of base64 characters to keep in logging payloads.
# Data URIs exceeding this are replaced with a size placeholder.
@ -389,7 +395,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@ -1613,6 +1619,7 @@ CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVA
MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
INTERNAL_CALLBACK_NAMES: Final = frozenset({"cache", "vector_store_pre_call_hook"})
# Headers to control callbacks
X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
@ -1629,6 +1636,7 @@ INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
"Truncation is a DB storage safeguard. "
@ -1856,6 +1864,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))
@ -1895,6 +1904,7 @@ _background_interaction_cost_polling_env: Final = os.getenv(
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
).lower()
BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS: Final = 300.0
PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)
PROXY_BATCH_WRITE_AT: Final = get_env_int("PROXY_BATCH_WRITE_AT", 10) # in seconds, increased from 10
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

1
litellm/containers/README.md
View file

@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
self,
headers: dict,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""Azure uses api-key header instead of Bearer token."""
import litellm

137
litellm/cost_calculator.py
View file

@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from httpx import Response
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import (
from litellm.llms.openai.cost_calculation import (
video_output_cost_per_second,
)
from litellm.llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTAudioTranscriptionConfig,
)
from litellm.llms.perplexity.cost_calculator import (
cost_per_token as perplexity_cost_per_token,
)
@ -147,6 +150,7 @@ from litellm.utils import (
TranscriptionResponse,
cached_get_model_info_helper,
token_counter,
unmapped_model_generalization,
)
if TYPE_CHECKING:
@ -325,6 +329,15 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return False
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -863,6 +876,16 @@ def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | Non
return own_model_info["litellm_provider"]
def _pricing_provider_for_cost_candidate(
candidate_provider: str | None,
request_provider: str | None,
current_provider: object,
) -> str | None:
if candidate_provider != request_provider:
return candidate_provider
return current_provider if isinstance(current_provider, str) else None
def select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -1426,32 +1449,15 @@ def completion_cost(
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
) -> float:
"""
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
Return the USD cost of a LiteLLM call.
Parameters:
completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request.
Pass `completion_response` to price a response LiteLLM returned. Without one, pass `model` plus
`messages` / `prompt` and `completion`, and token counts are computed from the text. Prices come from
`litellm.model_cost` unless `custom_cost_per_token` or `custom_cost_per_second` is given. `total_time`
is in milliseconds and only matters for per-second pricing such as Replicate and SageMaker.
[OPTIONAL PARAMS]
model (str): Optional. The name of the language model used in the completion calls
prompt (str): Optional. The input prompt passed to the llm
completion (str): Optional. The output completion text from the llm
total_time (float, int): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds
custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
Returns:
float: The cost in USD dollars for the completion based on the provided parameters.
Exceptions:
Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
Note:
- If completion_response is provided, the function extracts token information and the model name from it.
- If completion_response is not provided, the function calculates token counts based on the model and input text.
- The cost is calculated based on the model, prompt tokens, and completion tokens.
- For certain models containing "togethercomputer" in the name, prices are based on the model size.
- For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
or add it to model_prices_and_context_window.json.
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
@ -1549,14 +1555,20 @@ def completion_cost(
)
requested_model: Final = model
potential_model_names: Final = [
selected_model,
_get_response_model(completion_response),
]
if model is not None:
potential_model_names.append(model)
request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
raw_candidates: Final = (
(
selected_model,
base_model_provider if base_model_provider is not None else request_pricing_provider,
),
(_get_response_model(completion_response), request_pricing_provider),
*(((model, request_pricing_provider),) if model is not None else ()),
)
potential_model_names: Final = (
raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
)
for idx, model in enumerate(potential_model_names):
for idx, (model, pricing_provider) in enumerate(potential_model_names):
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("selected model name for cost calculation: %s", model)
@ -1739,6 +1751,12 @@ def completion_cost(
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):
@ -1895,13 +1913,6 @@ def completion_cost(
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
pricing_provider: str | None = (
base_model_provider
if base_model_provider is not None and idx == 0
else custom_llm_provider
if isinstance(custom_llm_provider, str)
else None
)
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
@ -1909,7 +1920,11 @@ def completion_cost(
model=model,
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
@ -2008,7 +2023,11 @@ def completion_cost(
if cost_per_token_usage_object is not None and model:
_token_type_breakdown = get_token_type_cost_breakdown(
model=model,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
usage=cost_per_token_usage_object,
service_tier=service_tier,
data_residency=data_residency,
@ -2048,7 +2067,7 @@ def completion_cost(
)
if idx == len(potential_model_names) - 1:
raise e
raise Exception(f"Unable to calculat cost for received potential model names - {potential_model_names}")
raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}")
except Exception as e:
raise e
@ -2063,13 +2082,20 @@ def get_response_cost_from_hidden_params(
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
response_cost: Final = additional_headers["llm_provider-x-litellm-response-cost"]
if response_cost is None:
return None
return float(additional_headers["llm_provider-x-litellm-response-cost"])
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def _billable_header_cost(value: object) -> float | None:
if not isinstance(value, (str, int, float)):
return None
try:
cost: Final = float(value)
except ValueError:
return None
return cost if math.isfinite(cost) and cost >= 0 else None
def response_cost_calculator(
response_object: ModelResponse
| EmbeddingResponse
@ -2263,6 +2289,25 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn
return None
_CostCandidate: TypeAlias = tuple[str | None, str | None]
def _is_unpriced_generalized_candidate(candidate: _CostCandidate) -> bool:
model, pricing_provider = candidate
if model is None:
return False
rule_info: Final = unmapped_model_generalization(model, pricing_provider)
return rule_info is not None and not _cost_map_entry_prices_anything(rule_info)
def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, ...]) -> tuple[_CostCandidate, ...]:
unpriced: Final = tuple(_is_unpriced_generalized_candidate(candidate) for candidate in candidates)
return (
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if not is_unpriced),
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if is_unpriced),
)
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None

10
litellm/decisions/call.py
View file

@ -57,7 +57,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.utils import ProviderConfigManager
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
@ -458,7 +458,7 @@ def prepare_call(
) from error
try:
resolved_api_base, resolved_api_key = provider_config.resolve_connection(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or api_base, dynamic_api_key or api_key
)
except ValueError as error:
@ -469,6 +469,7 @@ def prepare_call(
model=model,
llm_provider=provider,
)
resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
if resolved_api_key is None and provider_config.api_key_required:
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",
@ -666,7 +667,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)

31
litellm/decisions/main.py
View file

@ -1,5 +1,7 @@
import asyncio
import warnings
from collections.abc import Mapping
from functools import partial
from typing import Final
import httpx
@ -104,19 +106,22 @@ async def adecisions(
videos: DecisionsVideoClips | None = None,
**kwargs: object,
) -> OpenAIDecisionResponse | DecisionsResponse:
call: Final = _prepare_decisions_call(
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
call: Final = await asyncio.to_thread(
partial(
_prepare_decisions_call,
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return await asend(call)

6
litellm/exceptions.py
View file

@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
)
class MalformedRequestFieldError(BadRequestError):
pass
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,
@ -1217,6 +1221,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
num_retries: int | None = None,
generated_content: str = "",
is_pre_first_chunk: bool = False,
emitted_disqualifying_content: bool = False,
):
original_status: Final = getattr(original_exception, "status_code", None)
self.status_code = int(original_status) if original_status is not None else 503
@ -1229,6 +1234,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.num_retries = num_retries
self.generated_content = generated_content
self.is_pre_first_chunk = is_pre_first_chunk
self.emitted_disqualifying_content = emitted_disqualifying_content
# Create a response if one wasn't provided
if response is None:

14
litellm/experimental_mcp_client/client.py
View file

@ -498,6 +498,7 @@ class MCPClient:
elicitation_callback: "ElicitationCallback | None" = None,
logging_callback: Callable | None = None,
protocol_version: MCPUpstreamProtocol = "auto",
upstream_transport: httpx2.AsyncBaseTransport | None = None,
):
self.protocol_version: MCPUpstreamProtocol = TypeAdapter[MCPUpstreamProtocol](
MCPUpstreamProtocol
@ -524,6 +525,7 @@ class MCPClient:
self._sampling_callback: Callable | None = sampling_callback
self._elicitation_callback: ElicitationCallback | None = elicitation_callback
self._logging_callback: Callable | None = logging_callback
self._upstream_transport: httpx2.AsyncBaseTransport | None = upstream_transport
self._session_callbacks_override: _SessionCallbackMap | None = None
# handle the basic auth value if provided
if auth_value:
@ -610,7 +612,11 @@ class MCPClient:
return stdio_client(server_params), None
if self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
return (
sse_client(
url=self.server_url,
@ -622,7 +628,11 @@ class MCPClient:
)
# HTTP transport (default)
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
http_client = httpx_client_factory(
headers=headers,

6
litellm/files/main.py
View file

@ -259,6 +259,7 @@ def create_file(
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
create_file_data=_create_file_request,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -388,6 +389,7 @@ def file_retrieve(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -569,6 +571,7 @@ def file_delete(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -775,6 +778,7 @@ def file_list(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -972,6 +976,7 @@ def file_content(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -1093,6 +1098,7 @@ def file_content_streaming(
organization=openai_creds.organization,
chunk_size=chunk_size,
client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
if not _is_async:

4
litellm/fine_tuning/main.py
View file

@ -239,6 +239,7 @@ def create_fine_tuning_job(
client=kwargs.get(
"client", None
), # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -436,6 +437,7 @@ def cancel_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -591,6 +593,7 @@ def list_fine_tuning_jobs(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -730,6 +733,7 @@ def retrieve_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":

9
litellm/images/main.py
View file

@ -39,6 +39,7 @@ from openai.types.audio.transcription_create_params import FileTypes
# BFL handlers
from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
@ -505,6 +506,7 @@ def image_generation(
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":
if model is None:
@ -681,10 +683,13 @@ def image_variation(
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
if api_key is None and (
resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
is None
):
raise ValueError("API key is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,

19
litellm/integrations/custom_guardrail.py
View file

@ -695,9 +695,7 @@ class CustomGuardrail(CustomLogger):
supported_event_hooks: list[GuardrailEventHooks],
) -> None:
allowed_hooks: Final = frozenset(supported_event_hooks) | (
frozenset((GuardrailEventHooks.logging_only,))
if self.uses_apply_guardrail_interface() and not self.use_native_lifecycle_hooks
else frozenset()
frozenset((GuardrailEventHooks.logging_only,)) if self.accepts_logging_only_mode() else frozenset()
)
def _validate_event_hook_list_is_in_supported_event_hooks(
@ -869,8 +867,13 @@ class CustomGuardrail(CustomLogger):
return True
return False
def uses_apply_guardrail_interface(self) -> bool:
return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
@classmethod
def uses_apply_guardrail_interface(cls) -> bool:
return cls.apply_guardrail is not CustomGuardrail.apply_guardrail
@classmethod
def accepts_logging_only_mode(cls) -> bool:
return cls.uses_apply_guardrail_interface() and not cls.use_native_lifecycle_hooks
@classmethod
def supports_logging_only_scope(cls) -> bool:
@ -1029,7 +1032,7 @@ class CustomGuardrail(CustomLogger):
)
return kwargs, result
litellm_params: Final = kwargs.get("litellm_params") or {}
scratch_metadata: Final = {
scratch_metadata: Final[dict[str, object]] = { # mutable-ok: apply_guardrail records its verdict here
key: value
for key, value in (litellm_params.get("metadata") or {}).items()
if key != "standard_logging_guardrail_information"
@ -1083,11 +1086,11 @@ class CustomGuardrail(CustomLogger):
async def _scan_logged_call(
self,
kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract
kwargs: dict[str, object], # mutable-ok: CustomLogger.async_logging_hook contract
response: object | None,
translation: "BaseTranslation",
output_translation: "BaseTranslation",
scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata
scratch_metadata: dict[str, object], # mutable-ok: apply_guardrail records its verdict into request metadata
) -> None:
scratch_fields: Final = self._copy_scratch_request_fields(kwargs)
scratch_input, scratch_tools = scratch_fields or (None, None)

2
litellm/integrations/gitlab/gitlab_prompt_manager.py
View file

@ -219,7 +219,7 @@ class GitLabTemplateManager:
elif value.replace(".", "").isdigit():
try:
result[key] = float(value)
except Exception:
except ValueError:
result[key] = value
else:
result[key] = value.strip("\"'")

71
litellm/integrations/langfuse/langfuse.py
View file

@ -103,6 +103,43 @@ def _is_session_header_trace(trace_id: object, session_id: object, proxy_server_
)
def langfuse_trace_metadata(metadata: object, proxy_server_request: object) -> Mapping[str, object]:
entries: Final = _object_mapping(metadata) or _NO_METADATA
request: Final = _object_mapping(proxy_server_request)
headers: Final = _object_mapping(request.get("headers")) if request is not None else None
if headers is None:
return entries
header_entries: Final = {
key.removeprefix("langfuse_"): value
for key, value in _widened_items(headers)
if isinstance(key, str) and key.startswith("langfuse_")
}
return MappingProxyType({**entries, **header_entries})
def langfuse_trace_id(
metadata: Mapping[str, object],
standard_trace_id: object,
litellm_trace_id: object,
litellm_call_id: object,
proxy_server_request: object,
) -> object:
existing_trace_id: Final = metadata.get("existing_trace_id")
if existing_trace_id is not None:
return existing_trace_id
metadata_trace_id: Final = metadata.get("trace_id")
trace_id: Final = (
metadata_trace_id
if metadata_trace_id is not None
else standard_trace_id
if standard_trace_id is not None
else litellm_trace_id or litellm_call_id
)
if _is_session_header_trace(trace_id, metadata.get("session_id"), proxy_server_request):
return litellm_call_id or trace_id
return trace_id
class _UsageObject(Protocol):
"""Token-count surface the Langfuse logger reads off a response usage payload."""
@ -680,33 +717,19 @@ class LangFuseLogger:
# Add default langfuse tags
tags = self.add_default_langfuse_tags(tags=tags, kwargs=kwargs, metadata=metadata)
call_trace_id: Final = langfuse_trace_id(
metadata=clean_metadata,
standard_trace_id=standard_logging_object.get("trace_id")
if standard_logging_object is not None
else None,
litellm_trace_id=kwargs.get("litellm_trace_id"),
litellm_call_id=litellm_call_id,
proxy_server_request=litellm_params.get("proxy_server_request"),
)
session_id: Final = clean_metadata.pop("session_id", None)
trace_name = cast(str | None, clean_metadata.pop("trace_name", None))
trace_id = clean_metadata.pop("trace_id", None)
# Use standard_logging_object.trace_id if available (when trace_id from metadata is None)
# This allows standard trace_id to be used when provided in standard_logging_object
if trace_id is None and standard_logging_object is not None:
trace_id = cast(str | None, standard_logging_object.get("trace_id"))
# Fallback to litellm_call_id if no trace_id found
if trace_id is None:
trace_id = kwargs.get("litellm_trace_id") or litellm_call_id
clean_metadata.pop("trace_id", None)
existing_trace_id: Final = clean_metadata.pop("existing_trace_id", None)
# If existing_trace_id is provided, use it as the trace_id to return
# This allows continuing an existing trace while still returning the correct trace_id
if existing_trace_id is not None:
trace_id = existing_trace_id
call_trace_id: Final = (
litellm_call_id or trace_id
if existing_trace_id is None
and _is_session_header_trace(trace_id, session_id, litellm_params.get("proxy_server_request"))
else trace_id
)
if call_trace_id != trace_id:
verbose_logger.debug(
"Langfuse: trace_id %s came from a session header; using call id %s so each call gets its own trace",
trace_id,
call_trace_id,
)
requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ()))
update_trace_keys: Final = (
requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else ()

32
litellm/integrations/prometheus.py
View file

@ -27,6 +27,7 @@ from litellm.constants import (
PROXY_REJECTED_BEFORE_ROUTING_KEY,
)
from litellm.exceptions import (
MalformedRequestFieldError,
validate_rate_limit_category,
validate_rate_limit_type,
)
@ -258,6 +259,7 @@ _POSITIVE_SERIES_CAP: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field
_POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(gt=0)])
_SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
_DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
_ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@ -269,6 +271,15 @@ def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _Series
return None
def _gauge_reading_or_none(value: object) -> float | None:
if not isinstance(value, str | int | float):
return None
try:
return float(value)
except (ValueError, OverflowError):
return None
def _positive_or_ignored(setting: str, value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
if value is None:
return None
@ -3104,7 +3115,11 @@ class PrometheusLogger(CustomLogger):
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
# into requested_model and leave deployment-scoped labels empty.
deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
deployment_selected: Final = (
bool(model_id)
and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
and not isinstance(exception, MalformedRequestFieldError)
)
if deployment_selected:
label_litellm_model_name = litellm_model_name
label_model_id = model_id
@ -3345,12 +3360,11 @@ class PrometheusLogger(CustomLogger):
model_group=enum_values.model_group,
)
remaining_requests: int | None = None
remaining_tokens: int | None = None
if additional_headers := standard_logging_payload["hidden_params"]["additional_headers"]:
# OpenAI / OpenAI Compatible headers
remaining_requests = additional_headers.get("x_ratelimit_remaining_requests", None)
remaining_tokens = additional_headers.get("x_ratelimit_remaining_tokens", None)
additional_headers: Final = _ADDITIONAL_HEADERS.validate_python(
standard_logging_payload["hidden_params"]["additional_headers"] or {}
)
remaining_requests: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_requests"))
remaining_tokens: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_tokens"))
if litellm_overhead_time_ms := standard_logging_payload["hidden_params"].get("litellm_overhead_time_ms"):
_labels = prometheus_label_factory(
@ -3368,7 +3382,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
if remaining_requests:
if remaining_requests is not None:
"""
"model_group",
"api_provider",
@ -3382,7 +3396,7 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests)
if remaining_tokens:
if remaining_tokens is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_remaining_tokens_metric"),
enum_values=enum_values,

289
litellm/interactions/background_cost_polling.py
View file

@ -32,19 +32,24 @@ task to settle. A ``BackgroundSettlementStore`` makes the pending settlement
durable across processes: the create registers the request context that
billing needs (never provider credentials), the settlement is claimed
exactly once through the store, and a delete on any replica rebuilds the
billing context from the store when the poll task is not local. Rows left
unclaimed by a process that died are resumed at startup. The default store is
in-memory, which keeps the SDK and single-process behavior unchanged; the
proxy installs a database-backed one.
billing context from the store when the poll task is not local. Every row
still unsettled when a process starts is resumed there, including one whose
settler died between its claim and its outcome: the database-backed store
treats a claim as a lease that its settler renews until it records the
outcome, so such a row is claimable again once the lease runs out. The
default store is in-memory, which keeps the SDK and single-process behavior
unchanged; the proxy installs a database-backed one.
"""
import asyncio
import itertools
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
from openai import APIStatusError
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
from pydantic_core import PydanticSerializationError, to_jsonable_python
@ -54,10 +59,12 @@ from litellm.constants import (
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.types.interactions import InteractionsAPIResponse
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import CredentialLiteLLMParams
from litellm.types.utils import CustomPricingLiteLLMParams
if TYPE_CHECKING:
@ -101,6 +108,7 @@ class PendingBackgroundInteraction:
custom_llm_provider: str
create_context: BackgroundInteractionCreateContext
created_at: datetime
claim_expires_at: datetime | None = None
class BackgroundSettlementStore(Protocol):
@ -112,9 +120,11 @@ class BackgroundSettlementStore(Protocol):
async def claim(self, interaction_id: str) -> bool: ...
async def renew_claim(self, interaction_id: str) -> None: ...
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None: ...
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]: ...
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]: ...
@dataclass(frozen=True, slots=True)
@ -145,10 +155,13 @@ class InMemoryBackgroundSettlementStore:
self._rows[interaction_id] = None
return True
async def renew_claim(self, interaction_id: str) -> None:
return None
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
return None
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
return tuple(row for row in self._rows.values() if row is not None)
@ -169,11 +182,15 @@ class BackgroundInteractionPollContext:
interaction_id: str
custom_llm_provider: str
logging_obj: "LiteLLMLoggingObj"
api_key: str | None = None
api_base: str | None = None
credentials: CredentialLiteLLMParams = field(default_factory=CredentialLiteLLMParams)
deployment_id: str | None = None
model_group: str | None = None
initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
start_delay_seconds: float = 0.0
outcome_write_timeout_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 2
claim_renewal_interval_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 3
store: BackgroundSettlementStore = field(default_factory=InMemoryBackgroundSettlementStore)
resumed: bool = False
@ -184,15 +201,41 @@ FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Await
async def fetch_background_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
from litellm.interactions import aget
credentials: Final = context.credentials
return await aget(
interaction_id=context.interaction_id,
custom_llm_provider=context.custom_llm_provider,
api_key=context.api_key,
api_base=context.api_base,
api_key=credentials.api_key,
api_base=credentials.api_base,
vertex_project=credentials.vertex_project,
vertex_location=credentials.vertex_location,
vertex_credentials=credentials.vertex_credentials,
**{"no-log": True},
)
_FETCH_CREDENTIAL_FIELDS: Final = ("api_key", "api_base", "vertex_project", "vertex_location", "vertex_credentials")
def _fetch_credentials(request_kwargs: Mapping[str, object]) -> CredentialLiteLLMParams:
try:
return CredentialLiteLLMParams.model_validate(
{field_name: request_kwargs.get(field_name) for field_name in _FETCH_CREDENTIAL_FIELDS}
)
except ValidationError:
verbose_logger.debug(
"Fetching a background interaction with environment credentials: the request's are malformed"
)
return CredentialLiteLLMParams()
_PERMANENT_FETCH_STATUS_CODES: Final = frozenset({400, 401, 403, 404})
def _is_permanent_fetch_error(error: Exception) -> bool:
return isinstance(error, APIStatusError) and error.status_code in _PERMANENT_FETCH_STATUS_CODES
def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
elapsed = 0.0 # rebind-ok: the schedule accumulates the time it has already yielded
interval = initial # rebind-ok: the schedule doubles the interval up to the cap
@ -231,6 +274,7 @@ _UNCARRIED_METADATA_KEY: Final = "user_api_key_auth"
_JSON_VALUE: Final = TypeAdapter(JsonValue)
_STRING: Final = TypeAdapter(str)
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECT_DICT: Final = TypeAdapter(dict[str, object])
def _carries(key: str) -> bool:
@ -256,10 +300,36 @@ def _as_datetime(start_time: datetime | float) -> datetime:
return start_time if isinstance(start_time, datetime) else datetime.fromtimestamp(start_time, tz=timezone.utc)
def _spend_metadata(logging_obj: "LiteLLMLoggingObj") -> Mapping[str, object]:
"""
The metadata the creating worker's own spend row is built from, with the
request tags computed once, header-derived ones included, because a
rebuilt logging object has no request headers to derive them from.
"""
litellm_params: Final = _OBJECT_DICT.validate_python(logging_obj.model_call_details.get("litellm_params") or {})
return {
**_OBJECT_MAPPING.validate_python(get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)),
**_request_tags(litellm_params),
}
def _request_tags(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
try:
return {
"tags": StandardLoggingPayloadSetup.get_request_tags(
litellm_params=_OBJECT_DICT.validate_python(litellm_params),
proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}),
)
}
except AttributeError:
verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list")
return {}
def _create_context(logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str) -> BackgroundInteractionCreateContext:
metadata: Final = _OBJECT_MAPPING.validate_python(
get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
)
metadata: Final = _spend_metadata(logging_obj)
litellm_params: Final = _OBJECT_MAPPING.validate_python(logging_obj.litellm_params)
model: Final = logging_obj.model_call_details.get("model")
return BackgroundInteractionCreateContext(
@ -313,6 +383,16 @@ async def _settled_elsewhere(context: BackgroundInteractionPollContext) -> bool:
return False
async def _settled_elsewhere_before_the_window_ends(
context: BackgroundInteractionPollContext, intervals: Iterator[float]
) -> bool:
for interval in intervals:
await asyncio.sleep(interval)
if await _settled_elsewhere(context):
return True
return False
async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
"""
Exactly-once gate between every settler of one interaction, on every
@ -326,30 +406,118 @@ async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
return None
async def _record(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> SettlementOutcome:
async def _wrote_outcome(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
try:
await context.store.record_outcome(context.interaction_id, outcome)
except Exception: # noqa: BLE001 # the outcome is an audit trail; the claim already made the settlement exclusive
verbose_logger.exception("Could not record the settlement of background interaction %s", context.interaction_id)
except Exception as e: # noqa: BLE001 # a failed write is retried while the claim still shuts out takeovers
verbose_logger.warning(
"Could not record the settlement of background interaction %s: %s", context.interaction_id, e
)
return False
return True
async def _recorded(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
"""
Retries a failed write for half the claim lease, while no other settler
can take the row over, so a database blip does not leave the row to a
takeover or hold the budget reservation that only a recorded release frees.
"""
retries: Final = _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.outcome_write_timeout_seconds,
)
for delay in itertools.chain((0.0,), retries):
await asyncio.sleep(delay)
if await _wrote_outcome(context, outcome):
return True
verbose_logger.error(
"Could not record the settlement of background interaction %s within %ss; the next process to take "
"over its lapsed claim settles it",
context.interaction_id,
context.outcome_write_timeout_seconds,
)
return False
async def _bill(context: BackgroundInteractionPollContext, response: InteractionsAPIResponse) -> SettlementOutcome:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
await _recorded(context, "billed")
return "billed"
async def _release(
context: BackgroundInteractionPollContext, outcome: Literal["released", "unsettled"]
) -> SettlementOutcome:
"""
A release writes no spend row, so a takeover cannot tell whether the
settler before it released. The outcome is recorded first and the
reservation released only once it is, which leaves a takeover with
nothing to release twice.
"""
if await _recorded(context, outcome):
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return outcome
async def _renew_claim_until_cancelled(context: BackgroundInteractionPollContext) -> None:
while True:
await asyncio.sleep(context.claim_renewal_interval_seconds)
try:
await context.store.renew_claim(context.interaction_id)
except Exception as e: # noqa: BLE001 # a missed renewal is retried next interval, well inside the lease
verbose_logger.warning(
"Could not renew the settlement claim of background interaction %s: %s", context.interaction_id, e
)
async def _settle(
context: BackgroundInteractionPollContext, response: InteractionsAPIResponse | None
) -> SettlementOutcome:
"""
Billing runs every success callback in turn, and a slow one can outlast
the claim lease, so the claim is renewed until the outcome is recorded.
A settler that stops renewing has died, and only then can another take
the row over.
"""
renewal: Final = asyncio.create_task(_renew_claim_until_cancelled(context))
try:
if response is not None and response.status in _TERMINAL_STATUSES and response.usage is not None:
return await _bill(context, response)
return await _release(context, "released")
finally:
renewal.cancel()
async def poll_and_log_background_interaction_cost(
context: BackgroundInteractionPollContext,
fetch_interaction: FetchInteraction = fetch_background_interaction,
) -> SettlementOutcome | None:
last_response: InteractionsAPIResponse | None = None # rebind-ok: the give-up path settles from the last poll
for interval in _poll_intervals(
await asyncio.sleep(context.start_delay_seconds)
intervals: Final = _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.timeout_seconds,
):
)
for interval in intervals:
await asyncio.sleep(interval)
if await _settled_elsewhere(context):
return None
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop
if context.resumed and _is_permanent_fetch_error(e):
verbose_logger.warning(
"Stopped fetching resumed background interaction %s: the provider answered %s, so this "
"process cannot read it and leaves it to the replica that created it until the poll window ends",
context.interaction_id,
e,
)
if await _settled_elsewhere_before_the_window_ends(context, intervals):
return None
break
verbose_logger.debug(
"Background interaction cost poll for %s failed, will retry: %s",
context.interaction_id,
@ -363,11 +531,11 @@ async def poll_and_log_background_interaction_cost(
continue
if not claimed:
return None
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
return await _settle(context, response)
if not await _claim(context):
return None
if last_response is not None and last_response.status in _TERMINAL_STATUSES:
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=last_response))
return await _settle(context, last_response)
if last_response is not None and last_response.status not in _POLLABLE_STATUSES:
verbose_logger.error(
"Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
@ -379,20 +547,10 @@ async def poll_and_log_background_interaction_cost(
)
else:
verbose_logger.warning(
"Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
"Gave up cost polling for background interaction %s; its usage will not be tracked",
context.interaction_id,
context.timeout_seconds,
)
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return await _record(context, "unsettled")
async def _settle_terminal(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> SettlementOutcome:
if response.status in _TERMINAL_STATUSES and response.usage is not None:
await _bill_settled_interaction(logging_obj=logging_obj, response=response)
return "billed"
await _release_open_budget_reservation(logging_obj=logging_obj)
return "released"
return await _release(context, "unsettled")
async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
@ -405,7 +563,9 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") ->
settlement must release the reservation here or the spend counters stay
pinned at the estimated cost.
"""
metadata: Final = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
metadata: Final = _OBJECT_MAPPING.validate_python(
get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
)
budget_reservation: Final = metadata.get("user_api_key_budget_reservation")
if not isinstance(budget_reservation, dict):
return
@ -518,13 +678,18 @@ class _UnverifiedRegistrationStore:
return False
return await self.local.claim(interaction_id)
async def renew_claim(self, interaction_id: str) -> None:
if await self.local.is_claimed(interaction_id):
return
await self.durable.renew_claim(interaction_id)
async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
if await self.local.is_claimed(interaction_id):
return
await self.durable.record_outcome(interaction_id, outcome)
async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
return await self.durable.unclaimed()
async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
return await self.durable.unsettled()
async def _registered_store(
@ -559,8 +724,6 @@ async def maybe_schedule_background_interaction_cost_polling(
logging_obj: Final = create_kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return None
api_key: Final = create_kwargs.get("api_key")
api_base: Final = create_kwargs.get("api_base")
pending: Final = PendingBackgroundInteraction(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
@ -571,8 +734,7 @@ async def maybe_schedule_background_interaction_cost_polling(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
credentials=_fetch_credentials(create_kwargs),
store=await _registered_store(store or _STORE.store, pending),
)
return _track_poll(context, fetch_interaction)
@ -603,10 +765,7 @@ async def _settle_before_delete(
) -> SettlementOutcome | None:
if not await _claim(context):
return None
if response is None:
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return await _record(context, "released")
return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
return await _settle(context, response)
async def maybe_settle_background_interaction_before_delete(
@ -622,14 +781,11 @@ async def maybe_settle_background_interaction_before_delete(
pending: Final = await _pending(settlement_store, interaction_id)
if pending is None:
return None
api_key: Final = delete_kwargs.get("api_key")
api_base: Final = delete_kwargs.get("api_base")
context: Final = BackgroundInteractionPollContext(
interaction_id=interaction_id,
custom_llm_provider=pending.custom_llm_provider,
logging_obj=_rebuild_logging_obj(pending.create_context),
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
credentials=_fetch_credentials(delete_kwargs),
store=settlement_store,
)
try:
@ -644,9 +800,9 @@ async def maybe_settle_background_interaction_before_delete(
return await _settle_before_delete(context, response)
async def _unclaimed(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
async def _unsettled(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
try:
return await store.unclaimed()
return await store.unsettled()
except Exception: # noqa: BLE001 # an unreadable store at startup leaves its rows for the next boot
verbose_logger.exception("Could not list the unsettled background interactions")
return ()
@ -662,17 +818,36 @@ class PollSchedule:
DEFAULT_POLL_SCHEDULE: Final = PollSchedule()
def _deployment_id(create_context: BackgroundInteractionCreateContext) -> str | None:
model_info: Final = create_context.metadata.get("model_info")
deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None
return deployment_id if isinstance(deployment_id, str) else None
def _model_group(create_context: BackgroundInteractionCreateContext) -> str | None:
model_group: Final = create_context.metadata.get("model_group")
return model_group if isinstance(model_group, str) else None
def _seconds_until(moment: datetime | None) -> float:
if moment is None:
return 0.0
return max((moment - datetime.now(timezone.utc)).total_seconds(), 0.0)
def _resumed_context(
row: PendingBackgroundInteraction, store: BackgroundSettlementStore, schedule: PollSchedule
) -> BackgroundInteractionPollContext:
age_seconds: Final = (datetime.now(timezone.utc) - row.created_at).total_seconds()
return BackgroundInteractionPollContext(
interaction_id=row.interaction_id,
custom_llm_provider=row.custom_llm_provider,
logging_obj=_rebuild_logging_obj(row.create_context),
deployment_id=_deployment_id(row.create_context),
model_group=_model_group(row.create_context),
initial_interval_seconds=schedule.initial_interval_seconds,
max_interval_seconds=schedule.max_interval_seconds,
timeout_seconds=max(schedule.timeout_seconds - age_seconds, schedule.initial_interval_seconds),
timeout_seconds=schedule.timeout_seconds,
start_delay_seconds=_seconds_until(row.claim_expires_at),
store=store,
resumed=True,
)
@ -684,13 +859,15 @@ async def resume_unsettled_background_interactions(
schedule: PollSchedule = DEFAULT_POLL_SCHEDULE,
) -> tuple["asyncio.Task[SettlementOutcome | None]", ...]:
"""
Pick up every settlement no process has claimed, which is what a replica
that died mid-poll leaves behind. Each resumed poll keeps the remaining
share of the original timeout and gets at least one fetch, so a completed
interaction is still billed however late the resume comes.
Pick up every settlement no process has finished, which is what a replica
that died mid-poll or mid-settlement leaves behind. A row whose claim is
still within its lease waits until the lease runs out, then polls like any
other. Each resumed poll gets the full poll window however old its row is,
so a fetch that fails right after a late restart is retried rather than
ending the row as unsettled.
"""
return tuple(
_track_poll(_resumed_context(row, store, schedule), fetch_interaction)
for row in await _unclaimed(store)
for row in await _unsettled(store)
if row.interaction_id not in _ACTIVE_POLLS
)

2
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -44,7 +44,7 @@ from litellm.litellm_core_utils.private_json import (
stage_private_json,
write_private_json,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
@dataclass(frozen=True, slots=True)

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -318,7 +318,7 @@ def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[in
items.pop(index)
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: dict) -> dict:
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: Mapping[str, object]) -> dict:
"""
Helper to get litellm metadata for spend tracking
@ -326,7 +326,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
and user_api_key values are in 'metadata'.
"""
potential_spend_tracking_metadata_substring: Final = "user_api_key"
for key, value in metadata.items():
for key, value in dict(metadata).items():
if potential_spend_tracking_metadata_substring in key:
litellm_metadata[key] = value
return litellm_metadata

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_auth_type",
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)

79
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -17,6 +17,7 @@ import random
import sys
import threading
import time
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, replace
from datetime import datetime, timezone
@ -24,11 +25,11 @@ from importlib.resources import files
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, Protocol
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm import verbose_logger
from litellm.constants import (
@ -266,6 +267,19 @@ class _SyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
class _S3Body(Protocol):
def read(self) -> bytes: ...
class _S3GetObjectResponse(TypedDict):
Body: ReadOnly[_S3Body]
ETag: NotRequired[ReadOnly[str]]
class _S3Client(Protocol):
def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
@ -276,6 +290,20 @@ def _default_reload_client() -> _AsyncGetClient:
return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
def _default_s3_client(timeout: int) -> _S3Client:
import boto3
from botocore.config import Config
return cast( # cast-ok: boto3's generic client factory does not expose its generated S3 response type
_S3Client,
boto3.client( # pyright: ignore[reportUnknownMemberType] # generic factory returns Unknown service clients
"s3",
region_name=os.getenv("AWS_REGION_NAME") or None,
config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
),
)
def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
reason: Final = f"{type(error).__name__} fetching {url}: {error}"
if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
@ -283,7 +311,41 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final[object] = json.loads(content)
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition
dict[str, object], parsed
)
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
parts: Final = urllib.parse.urlsplit(url)
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
try:
client: Final = _default_s3_client(timeout)
response: Final = client.get_object(Bucket=bucket, Key=key)
content: Final = response["Body"].read()
except (BotoCoreError, ClientError) as error:
return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
return _classify_fetch_content(content, url, response.get("ETag"))
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -292,6 +354,9 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
if parts.scheme == "s3":
return _attempt_fetch_s3(url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -307,15 +372,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
try:
parsed: Final = response.json()
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
return ModelCostMapReloaded(
model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag")
)
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
def _next_retry_wait(

5
litellm/litellm_core_utils/hidden_params.py
View file

@ -77,6 +77,11 @@ def get_hidden_params(obj: object) -> MutableMapping[str, object] | None:
return _as_hidden_params_mapping(hidden_params)
def served_from_cache(obj: object) -> bool:
hidden_params: Final = get_hidden_params(obj)
return hidden_params is not None and hidden_params.get("cache_hit") is True
def set_hidden_params(obj: object, hidden_params: dict[str, object] | HiddenParams) -> None:
if isinstance(obj, dict):
obj[_HIDDEN_PARAMS_ATTR] = hidden_params

396
litellm/litellm_core_utils/litellm_logging.py
View file

@ -41,6 +41,7 @@ from litellm.constants import (
DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
EMPTY_MAPPING,
INTERNAL_CALLBACK_NAMES,
PROVIDER_REQUEST_ID_HEADERS,
REDACTED_BY_LITELLM,
)
@ -79,7 +80,12 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_param
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
served_from_cache,
set_hidden_param,
)
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
is_unbilled_non_inference_call,
@ -2056,7 +2062,7 @@ class Logging(LiteLLMLoggingBaseClass):
) -> None:
if response_cost is None and not calculation_failed:
return
if self.model_call_details.get("cache_hit") is True:
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
self.model_call_details["zero_cost_diagnostic"] = None
return
try:
@ -2287,6 +2293,7 @@ class Logging(LiteLLMLoggingBaseClass):
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
for_async_request=True,
**kwargs,
)
@ -2295,6 +2302,54 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
def attempt_scoped_copy(self) -> "Logging":
"""A copy with its own model_call_details, so logging one attempt of a request through it leaves
the state the request's other attempts share untouched."""
attempt: Final = copy.copy(self)
attempt.model_call_details = dict(self.model_call_details)
return attempt
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""
usage: Final = self.model_call_details.pop("combined_usage_object", None)
response_cost: Final = self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
)
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
self.model_call_details["continued_partial_usage"] = usage
self.model_call_details["continued_partial_cost"] = response_cost
self.model_call_details["continued_response_id"] = response_id
def _add_continued_partial_spend(self, response: object) -> None:
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
own_usage: Final = getattr(response, "usage", None)
own_cost: Final = self.model_call_details.get("response_cost")
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
return
from litellm.cost_calculator import BaseTokenUsageProcessor
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, partial_usage]
)
if not isinstance(own_cost, (int, float)):
return
self.model_call_details["response_cost"] = own_cost + partial_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
total_cost=own_cost + partial_cost,
additional_costs={
**self.cost_breakdown.get("additional_costs", {}),
"mid_stream_partial_cost": partial_cost,
},
)
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
@ -2480,7 +2535,8 @@ class Logging(LiteLLMLoggingBaseClass):
start_time,
end_time,
build_logging_payload: bool = True,
):
publish_logging_payload: bool = True,
) -> StandardLoggingPayload | None:
"""Resolve hidden params, compute response cost, and emit the standard logging payload."""
self._surface_response_headers_from_result(logging_result)
hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {})
@ -2507,17 +2563,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
if not build_logging_payload:
return
return None
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logging_result, start_time, end_time
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
standard_logging_payload: Final = self._build_standard_logging_payload(
logging_result, start_time, end_time, isolate_call_details=not publish_logging_payload
)
if not publish_logging_payload:
return standard_logging_payload
self.model_call_details["standard_logging_object"] = standard_logging_payload
if standard_logging_payload is not None:
emit_standard_logging_payload(standard_logging_payload)
return standard_logging_payload
async def _prepare_baseline_cache_estimate(self, response_obj: object) -> None:
if self.baseline_cache_context is None:
@ -2533,12 +2589,16 @@ class Logging(LiteLLMLoggingBaseClass):
await invalidate_baseline_cache(self, reason, completed=completed)
def _build_standard_logging_payload(
self, init_response_obj: object, start_time: dt_object, end_time: dt_object
self,
init_response_obj: object,
start_time: dt_object,
end_time: dt_object,
isolate_call_details: bool = False,
) -> StandardLoggingPayload | None:
"""Build StandardLoggingPayload and accumulate its construction time."""
_start: Final = time.time()
payload: Final = get_standard_logging_object_payload(
kwargs=self.model_call_details,
kwargs=dict(self.model_call_details) if isolate_call_details else self.model_call_details,
init_response_obj=init_response_obj,
start_time=start_time,
end_time=end_time,
@ -2555,11 +2615,16 @@ class Logging(LiteLLMLoggingBaseClass):
emit_standard_logging_payload(stored_payload)
def _transform_usage_objects(self, result: object) -> object:
return self._transform_usage_objects_for_payload(result, self.model_call_details.get("standard_logging_object"))
def _transform_usage_objects_for_payload(
self, result: object, standard_logging_payload: StandardLoggingPayload | None
) -> object:
if isinstance(result, ResponsesAPIResponse):
result = result.model_copy()
transformed_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(result.usage)
setattr(result, "usage", transformed_usage)
if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
if standard_logging_payload is not None:
response_dict: Final = result.model_dump() if hasattr(result, "model_dump") else dict(result)
# Ensure usage is properly included with transformed chat format
if transformed_usage is not None:
@ -2568,7 +2633,7 @@ class Logging(LiteLLMLoggingBaseClass):
if hasattr(transformed_usage, "model_dump")
else dict(transformed_usage)
)
standard_logging_payload["response"] = response_dict
standard_logging_payload["response"] = response_dict # rebind-ok: mirrors the transformed usage
elif isinstance(result, TranscriptionResponse):
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
@ -2590,6 +2655,15 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"model": model})
def _with_continued_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
response_id: Final = self.model_call_details.get("continued_response_id")
if not isinstance(response_id, str) or getattr(response, "id", None) == response_id:
return response
return response.model_copy(update={"id": response_id})
def _success_handler_helper_fn(
self,
result: object = None,
@ -2599,6 +2673,27 @@ class Logging(LiteLLMLoggingBaseClass):
standard_logging_object: StandardLoggingPayload | None = None,
build_logging_payload: bool = True,
) -> tuple[datetime.datetime, datetime.datetime, object]:
prepared: Final = self._prepare_success_logging(
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
standard_logging_object=standard_logging_object,
build_logging_payload=build_logging_payload,
publish_logging_payload=True,
)
return prepared[0], prepared[1], prepared[2]
def _prepare_success_logging(
self,
result: object,
start_time: datetime.datetime | None,
end_time: datetime.datetime | None,
cache_hit: bool | None,
standard_logging_object: StandardLoggingPayload | None,
build_logging_payload: bool,
publish_logging_payload: bool,
) -> tuple[datetime.datetime, datetime.datetime, object, StandardLoggingPayload | None]:
try:
if start_time is None:
start_time = self.start_time
@ -2623,35 +2718,29 @@ class Logging(LiteLLMLoggingBaseClass):
result = self._handle_a2a_response_logging(result=result)
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
self.model_call_details["cache_hit"] = True
if isinstance(result, Response) and isinstance(
logging_result, (ModelResponse, EmbeddingResponse, TextCompletionResponse)
):
result = logging_result
if standard_logging_object is None and result is not None and self.stream is not True:
if self._is_recognized_call_type_for_logging(logging_result=logging_result) or isinstance(
logging_result, (dict, list)
):
self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
build_logging_payload=build_logging_payload,
)
elif standard_logging_object is not None:
self.model_call_details["standard_logging_object"] = standard_logging_object
else:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
# established one. A stream that assembles into a response
# object recomputes the cost right after this; a pass-through
# stream cannot (its body is opaque) and carries the cost its
# upstream reported in the response headers, which an
# unconditional reset would discard.
self.model_call_details.setdefault("response_cost", None)
standard_logging_payload: Final = self._resolve_success_logging_payload(
result=result,
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
standard_logging_object=standard_logging_object,
build_logging_payload=build_logging_payload,
publish_logging_payload=publish_logging_payload,
)
result = self._transform_usage_objects(result=result)
result = (
self._transform_usage_objects(result=result)
if publish_logging_payload
else self._transform_usage_objects_for_payload(result, standard_logging_payload)
)
if (
litellm.max_budget
@ -2670,10 +2759,47 @@ class Logging(LiteLLMLoggingBaseClass):
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
return start_time, end_time, result
return start_time, end_time, result, standard_logging_payload
except Exception as e:
raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {e}")
def _resolve_success_logging_payload(
self,
result: object,
logging_result: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
standard_logging_object: StandardLoggingPayload | None,
build_logging_payload: bool,
publish_logging_payload: bool,
) -> StandardLoggingPayload | None:
if standard_logging_object is not None:
if not publish_logging_payload:
return standard_logging_object.copy()
self.model_call_details["standard_logging_object"] = standard_logging_object
return standard_logging_object
if result is None or self.stream is True:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
# established one. A stream that assembles into a response
# object recomputes the cost right after this; a pass-through
# stream cannot (its body is opaque) and carries the cost its
# upstream reported in the response headers, which an
# unconditional reset would discard.
self.model_call_details.setdefault("response_cost", None)
return None
if not self._is_recognized_call_type_for_logging(logging_result=logging_result) and not isinstance(
logging_result, (dict, list)
):
return None
return self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
build_logging_payload=build_logging_payload,
publish_logging_payload=publish_logging_payload,
)
def _is_recognized_call_type_for_logging(
self,
logging_result: object,
@ -2807,11 +2933,14 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
self.success_handler(result=complete_streaming_response)
async def async_flush_passthrough_collected_chunks(self, collector: "PassthroughStreamCollector"):
async def async_flush_passthrough_collected_chunks(self, collector: "PassthroughStreamCollector") -> bool:
"""Log the response the collector built from the stream, and say whether there was one to log."""
complete_streaming_response: Final = collector.build_logged_response(litellm_logging_obj=self)
if complete_streaming_response is not None:
await self.async_success_handler(result=complete_streaming_response)
if complete_streaming_response is None:
return False
await self.async_success_handler(result=complete_streaming_response)
return True
def _restore_correlation_context(self) -> None:
"""Restore trace_id/session_id contextvars to their pre-call value.
@ -2855,6 +2984,8 @@ class Logging(LiteLLMLoggingBaseClass):
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
cache_hit: bool | None = None,
*,
for_async_request: bool = False,
**kwargs: object, # kwargs-ok: forwarded to _success_handler_body
) -> None:
"""Restores trace_id/session_id contextvars once this attempt's own success
@ -2862,7 +2993,12 @@ class Logging(LiteLLMLoggingBaseClass):
try:
with post_response_phase():
return self._success_handler_body(
result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
for_async_request=for_async_request,
**kwargs,
)
finally:
self._restore_correlation_context()
@ -2873,17 +3009,20 @@ class Logging(LiteLLMLoggingBaseClass):
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
cache_hit: bool | None = None,
for_async_request: bool = False,
**kwargs: Any, # kwargs-ok: forwarded from success_handler
) -> None:
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
start_time, end_time, result = self._success_handler_helper_fn(
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
start_time=start_time,
end_time=end_time,
result=result,
cache_hit=cache_hit,
standard_logging_object=kwargs.get("standard_logging_object", None),
build_logging_payload=True,
publish_logging_payload=not for_async_request,
)
litellm_params = self.model_call_details.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
@ -2908,20 +3047,52 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
)
if standard_logging_payload is not None:
# Only emit for sync requests (async_success_handler handles async)
if is_sync_request:
emit_standard_logging_payload(standard_logging_payload)
callback_logging_payload: Final = (
self._build_standard_logging_payload(
self.model_call_details["complete_streaming_response"],
start_time,
end_time,
isolate_call_details=True,
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
if for_async_request and complete_streaming_response is not None
else request_logging_payload
)
shared_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
)
isolated_logging_payload: Final = (
callback_logging_payload
if callback_logging_payload is not None or shared_logging_payload is None
else shared_logging_payload.copy()
)
call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict
call_details = ( # rebind-ok: logging hooks may return a new dict
self.model_call_details
if not for_async_request
else (
dict(self.model_call_details)
if isolated_logging_payload is None
else {**self.model_call_details, "standard_logging_object": isolated_logging_payload}
)
if standard_logging_payload is not None:
# Only emit for sync requests (async_success_handler handles async)
if is_sync_request:
emit_standard_logging_payload(standard_logging_payload)
)
callbacks: Final = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
global_callbacks=litellm.success_callback,
@ -2929,7 +3100,7 @@ class Logging(LiteLLMLoggingBaseClass):
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
model_call_details=call_details,
result=result,
)
## LOGGING HOOK ##
@ -2939,7 +3110,7 @@ class Logging(LiteLLMLoggingBaseClass):
if (
callback.should_run_guardrail(
data=self.model_call_details,
data=call_details,
event_type=GuardrailEventHooks.logging_only,
)
is not True
@ -2947,18 +3118,21 @@ class Logging(LiteLLMLoggingBaseClass):
continue
self.shadow_eval_request_snapshot = None
self.model_call_details, result = callback.logging_hook(
kwargs=self.model_call_details,
call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict
kwargs=call_details,
result=result,
call_type=self.call_type,
)
elif isinstance(callback, CustomLogger):
self.model_call_details, result = callback.logging_hook(
kwargs=self.model_call_details,
call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict
kwargs=call_details,
result=result,
call_type=self.call_type,
)
if not for_async_request:
self.model_call_details = call_details
call_details_view: Final = MappingProxyType[str, object](call_details)
self.mark_logging_complete(event_type="sync_success")
for callback in callbacks:
try:
@ -2972,7 +3146,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "promptlayer" and promptLayerLogger is not None:
print_verbose("reaches promptlayer for logging!")
promptLayerLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -2980,7 +3154,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "supabase" and supabaseClient is not None:
print_verbose("reaches supabase for logging!")
kwargs = self.model_call_details
kwargs = call_details
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
if self.stream:
@ -2988,10 +3162,10 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches supabase for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
model = self._model_call_details_view["model"]
messages = self._model_call_details_view["messages"]
model = call_details_view["model"]
messages = call_details_view["messages"]
optional_params = kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {})
supabaseClient.log_event(
@ -3011,7 +3185,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "wandb" and weightsBiasesLogger is not None:
print_verbose("reaches wandb for logging!")
weightsBiasesLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3020,7 +3194,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "logfire" and logfireLogger is not None:
verbose_logger.debug("reaches logfire for success logging!")
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
@ -3030,10 +3204,10 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches logfire for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
logfireLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3044,7 +3218,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "lunary" and lunaryLogger is not None:
print_verbose("reaches lunary for logging!")
model = self.model
kwargs = self.model_call_details
kwargs = call_details
input = kwargs.get("messages", kwargs.get("input", None))
@ -3055,7 +3229,7 @@ class Logging(LiteLLMLoggingBaseClass):
if "complete_streaming_response" not in kwargs:
continue
else:
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
lunaryLogger.log_event(
type=type,
@ -3065,7 +3239,7 @@ class Logging(LiteLLMLoggingBaseClass):
input=input,
user_id=kwargs.get("user", None),
# user_props=self.model_call_details.get("user_props", None),
extra=self._model_call_details_view.get("optional_params", {}),
extra=call_details_view.get("optional_params", {}),
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3075,8 +3249,8 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "helicone" and heliconeLogger is not None:
print_verbose("reaches helicone for logging!")
model = self.model
messages = self._model_call_details_view["input"]
kwargs = self.model_call_details
messages = call_details_view["input"]
kwargs = call_details
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
if self.stream:
@ -3084,7 +3258,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches helicone for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
heliconeLogger.log_success(
model=model,
@ -3099,7 +3273,7 @@ class Logging(LiteLLMLoggingBaseClass):
global langFuseLogger
print_verbose("reaches langfuse for success logging!")
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
@ -3112,7 +3286,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches langfuse for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=langFuseLogger,
@ -3137,7 +3311,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "greenscale" and greenscaleLogger is not None:
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
@ -3150,7 +3324,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches greenscale for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
greenscaleLogger.log_event(
kwargs=kwargs,
@ -3161,7 +3335,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "athina" and athinaLogger is not None:
deep_copy = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
deep_copy[k] = v
athinaLogger.log_event(
kwargs=deep_copy,
@ -3172,7 +3346,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "traceloop":
deep_copy = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response":
deep_copy[k] = v
traceloopLogger.log_event(
@ -3188,11 +3362,11 @@ class Logging(LiteLLMLoggingBaseClass):
if s3Logger is None:
s3Logger = S3Logger()
if self.stream:
if "complete_streaming_response" in self.model_call_details:
if "complete_streaming_response" in call_details:
print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response")
s3Logger.log_event(
kwargs=self.model_call_details,
response_obj=self.model_call_details["complete_streaming_response"],
kwargs=call_details,
response_obj=call_details["complete_streaming_response"],
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
@ -3201,7 +3375,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose("S3Logger Logger: Got Stream Event - No complete stream response as yet")
else:
s3Logger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3215,19 +3389,17 @@ class Logging(LiteLLMLoggingBaseClass):
openMeterLogger = OpenMeterLogger()
if self.stream and complete_streaming_response is None:
openMeterLogger.log_stream_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self._model_call_details_view["complete_response"]
call_details["complete_response"] = call_details.get("complete_streaming_response", {})
result = call_details_view["complete_response"]
openMeterLogger.log_success_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3240,20 +3412,18 @@ class Logging(LiteLLMLoggingBaseClass):
): # custom logger class
if self.stream and complete_streaming_response is None:
callback.log_stream_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self._model_call_details_view["complete_response"]
call_details["complete_response"] = call_details.get("complete_streaming_response", {})
result = call_details_view["complete_response"]
callback.log_success_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3264,7 +3434,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose(f"success callbacks: Running Custom Callback Function - {callback}")
customLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3445,12 +3615,15 @@ class Logging(LiteLLMLoggingBaseClass):
self.model,
)
self.model_call_details["response_cost"] = None
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
@ -4102,8 +4275,18 @@ class Logging(LiteLLMLoggingBaseClass):
start_time,
end_time,
cache_hit,
for_async_request=True,
)
def handle_sync_cache_hit_success_callbacks(
self,
result: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
cache_hit: bool | None = None,
) -> None:
executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
def _should_run_sync_callbacks_for_async_calls(self) -> bool:
"""
Returns:
@ -4115,7 +4298,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
_filtered_success_callbacks = self._remove_internal_custom_logger_callbacks(_combined_sync_callbacks)
_filtered_success_callbacks = self._remove_internal_litellm_callbacks(_filtered_success_callbacks)
return len(_filtered_success_callbacks) > 0
return any(
not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES)
for callback in _filtered_success_callbacks # pyright: ignore[reportUnknownVariableType] # callback lists are untyped
)
def _should_run_sync_failure_callbacks_for_async_calls(self) -> bool:
"""
@ -4282,6 +4468,9 @@ class Logging(LiteLLMLoggingBaseClass):
row is built from this response rather than re-parsed from the served body."""
self.bridged_model_response = response
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
def billable_result(self, result: object) -> object:
"""The response the cost calculator prices: a /v1/messages body served through the chat
bridge is replaced by the ModelResponse its spend row is built from, anything else is
@ -4382,6 +4571,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
responses_provider_metadata,
)
served_id: Final = _provider_response_id(result)
@ -4406,7 +4596,9 @@ class Logging(LiteLLMLoggingBaseClass):
"usage-only ModelResponse to keep the spend_logs row.",
str(e),
)
model_response: Final = litellm.ModelResponse(id=served_id)
model_response: Final = litellm.ModelResponse(
id=served_id, **responses_provider_metadata(result.model_extra)
)
model_response.model = self.model
usage: Final = result.usage
if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage):
@ -6634,6 +6826,11 @@ def _get_status_fields(
return StandardLoggingPayloadStatusFields(llm_api_status=llm_api_status, guardrail_status=guardrail_status)
def _served_from_cache_at_no_charge(result: object) -> bool:
hidden_params: Final = get_hidden_params(result)
return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
def _extract_response_obj_and_hidden_params(
init_response_obj: object,
original_exception: Exception | None,
@ -6796,8 +6993,9 @@ def get_standard_logging_object_payload(
) # maintain backwards compatibility with old request body check
saved_cache_cost: float = 0.0
if cache_hit is True:
if cache_hit is True or served_from_cache(init_response_obj):
id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id
if cache_hit is True:
saved_cache_cost = (
logging_obj.response_cost_calculator(
result=init_response_obj,

13
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -858,9 +858,9 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
if "web_search_options" in kwargs:
web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
return WebSearchOptions(**web_search_options)
web_search_options: Final = kwargs.get("web_search_options")
if isinstance(web_search_options, dict):
return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
kwargs=kwargs, tool_type="web_search_preview"
@ -876,10 +876,9 @@ class StandardBuiltInToolCostTracking:
_get_web_search_options = get_web_search_options
@staticmethod
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
if "tools" in kwargs:
return cast(list[object], kwargs.get("tools", []))
return None
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
tools: Final = kwargs.get("tools")
return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
@staticmethod
def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:

72
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -251,14 +251,7 @@ async def convert_to_streaming_response_async(
setattr(
model_response_object,
"usage",
Usage(
completion_tokens=response_object["usage"].get("completion_tokens", 0),
prompt_tokens=response_object["usage"].get("prompt_tokens", 0),
total_tokens=response_object["usage"].get("total_tokens", 0),
completion_tokens_details=_cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
),
),
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -345,12 +338,10 @@ def convert_to_streaming_response(
model_response_object.choices = choice_list
if "usage" in response_object and response_object["usage"] is not None:
setattr(model_response_object, "usage", Usage())
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
model_response_object.usage.completion_tokens_details = _cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
setattr(
model_response_object,
"usage",
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -414,13 +405,28 @@ class _CachedMessageProviderFields(TypedDict, total=False):
class _CachedUsageDetails(TypedDict, total=False):
completion_tokens: ReadOnly[object]
prompt_tokens: ReadOnly[object]
total_tokens: ReadOnly[object]
completion_tokens_details: ReadOnly[object]
prompt_tokens_details: ReadOnly[object]
cache_read_input_tokens: ReadOnly[object]
cache_creation_input_tokens: ReadOnly[object]
class _PromptCacheTokenCounts(TypedDict, total=False):
cache_read_input_tokens: ReadOnly[int]
cache_creation_input_tokens: ReadOnly[int]
_CACHED_MESSAGE_PROVIDER_FIELDS: Final = TypeAdapter(_CachedMessageProviderFields)
_CACHED_USAGE_DETAILS: Final = TypeAdapter(_CachedUsageDetails)
_FIELD_MAP: Final = TypeAdapter(dict[str, object])
_COMPLETION_TOKENS_DETAILS: Final = TypeAdapter(CompletionTokensDetailsWrapper)
_PROMPT_TOKENS_DETAILS: Final = TypeAdapter(PromptTokensDetailsWrapper)
_PROMPT_CACHE_TOKEN_COUNTS: Final = TypeAdapter(_PromptCacheTokenCounts)
_TOKEN_COUNT: Final = TypeAdapter(int)
_PROMPT_CACHE_TOKEN_FIELDS: Final = ("cache_read_input_tokens", "cache_creation_input_tokens")
def _cached_provider_specific_fields(message: _CachedMessageProviderFields) -> Mapping[str, object] | None:
@ -438,6 +444,44 @@ def _cached_completion_tokens_details(usage: _CachedUsageDetails) -> CompletionT
return None
def _cached_prompt_tokens_details(usage: _CachedUsageDetails) -> PromptTokensDetailsWrapper | None:
details: Final = usage.get("prompt_tokens_details")
if not isinstance(details, dict):
return None
try:
return _PROMPT_TOKENS_DETAILS.validate_python(details)
except ValidationError:
return None
def _cached_token_count(count: object) -> int | None:
try:
return _TOKEN_COUNT.validate_python(count)
except ValidationError:
return None
def _cached_prompt_cache_token_counts(usage: _CachedUsageDetails) -> _PromptCacheTokenCounts:
return _PROMPT_CACHE_TOKEN_COUNTS.validate_python(
{
field: count
for field in _PROMPT_CACHE_TOKEN_FIELDS
if (count := _cached_token_count(usage.get(field))) is not None
}
)
def _replayed_usage(usage: _CachedUsageDetails) -> Usage:
return Usage(
completion_tokens=_cached_token_count(usage.get("completion_tokens")) or 0,
prompt_tokens=_cached_token_count(usage.get("prompt_tokens")) or 0,
total_tokens=_cached_token_count(usage.get("total_tokens")) or 0,
completion_tokens_details=_cached_completion_tokens_details(usage),
prompt_tokens_details=_cached_prompt_tokens_details(usage),
**_cached_prompt_cache_token_counts(usage),
)
def handle_invalid_parallel_tool_calls(
tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall],
) -> list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None:

13
litellm/litellm_core_utils/llm_response_utils/get_headers.py
View file

@ -1,8 +1,11 @@
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.constants import PROVIDER_REQUEST_ID_HEADERS
_HEADERS: Final = TypeAdapter(Mapping[str, str])
_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id")
@ -16,6 +19,16 @@ def get_provider_request_id(headers: object) -> str | None:
return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None)
def provider_request_id_headers(headers: Mapping[str, object] | None) -> httpx.Headers | None:
"""The provider request id headers in `headers` whose values are ASCII strings, or None when there are none."""
request_id_headers: Final = tuple(
(name, value)
for name, value in (headers or {}).items()
if name.lower() in PROVIDER_REQUEST_ID_HEADERS and isinstance(value, str) and value.isascii()
)
return httpx.Headers(request_id_headers) if request_id_headers else None
def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict:
"""

19
litellm/litellm_core_utils/model_param_helper.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from functools import lru_cache
from typing import ClassVar, Final
@ -22,6 +23,24 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequest
from litellm.types.rerank import RerankRequest
def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> object | None:
"""Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
"""
if "reasoningSummary" in optional_params:
return optional_params["reasoningSummary"]
if "reasoning_summary" in optional_params:
return optional_params["reasoning_summary"]
extra_body: Final = optional_params.get("extra_body")
if isinstance(extra_body, dict):
if "reasoningSummary" in extra_body:
return extra_body["reasoningSummary"]
if "reasoning_summary" in extra_body:
return extra_body["reasoning_summary"]
return None
class ModelParamHelper:
# Cached at class level — deterministic set built from static OpenAI type annotations
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

125
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -9,7 +9,8 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from itertools import groupby, islice
from graphlib import CycleError, TopologicalSorter
from itertools import chain, groupby, islice
from os import PathLike
from pathlib import Path
from types import MappingProxyType
@ -1426,6 +1427,123 @@ def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str,
frontier = tuple(child for node in frontier for child in _subschemas(node))
def schema_fragment_ref(ref: str) -> str:
return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref
def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool:
definitions: Final = MappingProxyType(dict(_root_definitions(schema)))
if not _refs_in(schema) <= definitions.keys():
return True
try:
tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order())
except CycleError:
return True
return False
def schema_union_property_count(schema: Mapping[str, object]) -> int:
return sum(1 for _, node in _property_schemas(schema) if _is_union(node))
def schema_optional_property_count(schema: Mapping[str, object]) -> int:
return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema))
def schema_property_count(schema: Mapping[str, object]) -> int:
return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties")))
def schema_grammar_branch_count(schema: Mapping[str, object]) -> int:
return sum(_grammar_branches(node) for node in _schema_nodes(schema))
def schema_has_open_object(schema: Mapping[str, object]) -> bool:
return any(_is_open_object(node) for node in _schema_nodes(schema))
def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool:
return any(
isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern)
for node in _schema_nodes(schema)
)
def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str)
)
def _may_be_string(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type is None or node_type == "string" or "string" in _list_at(node, "type")
def is_object_schema(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type == "object" or (isinstance(node_type, list) and "object" in node_type)
def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))
_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...])
def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]:
value: Final = container.get(key)
return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else ()
def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]:
if container is None:
return iter(())
return ((key, value) for key in container if (value := _schema_at(container, key)) is not None)
def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
for defs_key in ("$defs", "definitions"):
yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key)))
def _refs_in(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str)
)
def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema))
def _is_union(node: Mapping[str, object]) -> bool:
return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1
def _grammar_branches(node: Mapping[str, object]) -> int:
properties: Final = _schema_at(node, "properties")
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return 2**optional * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
properties: Final = _schema_at(node, "properties")
if properties is None:
return frozenset()
return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str))
def _is_open_object(node: Mapping[str, object]) -> bool:
additional_properties: Final = node.get("additionalProperties")
if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node:
return True
return is_object_schema(node) and "properties" not in node and additional_properties is not False
def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
for key, value in node.items():
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
@ -1894,6 +2012,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa
return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS)
def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
"""Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys."""
return _message_without_fields(message, frozenset({"prefix"}))
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
"""
Filters a value from a dictionary

194
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -9,10 +9,11 @@ import xml.etree.ElementTree as ET
from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
from enum import Enum
from types import MappingProxyType
from typing import Any, Final, TypeAlias, TypedDict, cast, overload
from typing import Annotated, Any, Final, TypeAlias, TypedDict, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
from pydantic import BaseModel, JsonValue, TypeAdapter
from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
from pydantic_core import ErrorDetails, PydanticCustomError
import litellm
import litellm.types
@ -2582,6 +2583,48 @@ def _with_native_redacted_data(
_AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues]
def _client_tool_result(message: AllMessageValues, replaced_server_tool_answers: frozenset[str]) -> bool:
return (
message_field(message, "role") in ("tool", "function")
and message_field(message, "tool_call_id") not in replaced_server_tool_answers
)
def _tool_result_later_in_turn(
messages: Sequence[AllMessageValues], replaced_server_tool_answers: frozenset[str]
) -> tuple[bool, ...]:
from_the_end: Final = itertools.accumulate(
reversed(messages),
lambda later, message: (
message_field(message, "role") in ("user", "tool", "function")
and (later or _client_tool_result(message, replaced_server_tool_answers))
),
initial=False,
)
return tuple(from_the_end)[::-1][1:]
def _empty_text_placeholder(message: AllMessageValues) -> AnthropicMessagesTextParam:
placeholder: Final = AnthropicMessagesTextParam(type="text", text=_EMPTY_TEXT_PLACEHOLDER)
add_cache_control_to_content(anthropic_content_element=placeholder, original_content_element=message)
return placeholder
def _cache_breakpoint_placeholder(
message: AllMessageValues, tool_result_follows: bool
) -> tuple[AnthropicMessagesTextParam, ...]:
if tool_result_follows or not message_field(message, "cache_control"):
return ()
return (_empty_text_placeholder(message),)
def _empty_user_turn_placeholder(span: Sequence[AllMessageValues]) -> AnthropicMessagesTextParam | None:
empty_user_message: Final = next(
(message for message in reversed(span) if message["role"] == "user" and not message.get("content")), None
)
return None if empty_user_message is None else _empty_text_placeholder(empty_user_message)
def anthropic_messages_pt(
messages: list[AllMessageValues],
model: str,
@ -2636,6 +2679,7 @@ def anthropic_messages_pt(
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
force_base64 = is_bedrock_invoke or is_vertex_ai
tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers)
msg_i = 0
while msg_i < len(messages):
user_content: list[AnthropicMessagesUserMessageValues] = []
@ -2648,8 +2692,12 @@ def anthropic_messages_pt(
ChatCompletionToolMessage | ChatCompletionUserMessage | ChatCompletionFunctionMessage
) = messages[msg_i]
if user_message_types_block["role"] == "user":
if isinstance(user_message_types_block["content"], list):
for m in user_message_types_block["content"]:
if not (user_block_content := user_message_types_block.get("content")):
user_content.extend(
_cache_breakpoint_placeholder(user_message_types_block, tool_result_later_in_turn[msg_i])
)
elif isinstance(user_block_content, list):
for m in user_block_content:
if m.get("type", "") == "image_url":
m = cast(ChatCompletionImageObject, m)
format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
@ -2720,10 +2768,10 @@ def anthropic_messages_pt(
_file_content_element,
)
)
elif isinstance(user_message_types_block["content"], str):
elif isinstance(user_block_content, str):
_anthropic_content_text_element: AnthropicMessagesTextParam = {
"type": "text",
"text": user_message_types_block["content"],
"text": user_block_content,
}
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_text_element,
@ -2747,6 +2795,8 @@ def anthropic_messages_pt(
if user_content:
new_messages.append({"role": "user", "content": user_content})
elif (placeholder := _empty_user_turn_placeholder(messages[init_msg_i:msg_i])) is not None:
new_messages.append({"role": "user", "content": [placeholder]})
# Track unique tool IDs in this merge block to avoid duplication
unique_tool_ids: set[str] = set()
@ -4253,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag
return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]])
def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock:
return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block
def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock:
"""Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it."""
if message["role"] != "assistant":
return message
blocks: Final = message["content"]
last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None)
return BedrockMessageBlock(
role="assistant",
content=[
_rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks)
],
)
def _ensure_document_messages_have_text(
contents: list[BedrockMessageBlock],
) -> list[BedrockMessageBlock]:
@ -4844,7 +4912,12 @@ class BedrockConverseMessagesProcessor:
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(
_rename_duplicate_bedrock_document_names(contents)
)
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
@staticmethod
def translate_thinking_blocks_to_reasoning_content_blocks(
@ -5253,7 +5326,10 @@ def _bedrock_converse_messages_pt(
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
@ -5428,14 +5504,14 @@ def _bedrock_tools_pt(
description = name
defs = parameters.pop("$defs", {})
defs_copy = copy.deepcopy(defs)
defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {}
# Expand $ref references in parameters using the definitions
# Note: We don't pre-flatten defs as that causes exponential memory growth
# with circular references (see issue #19098). unpack_defs handles nested
# refs recursively and correctly detects/skips circular references.
unpack_defs(parameters, defs_copy)
normalize_json_schema_custom_types_to_object(parameters)
if parameters.get("type") not in _valid_json_schema_root_types:
if not (isinstance(root_type := parameters.get("type"), str) and root_type in _valid_json_schema_root_types):
parameters["type"] = "object"
tool_block = cast(
BedrockToolBlock,
@ -5521,7 +5597,7 @@ def function_call_prompt(
]
def response_schema_prompt(model: str, response_schema: dict) -> str:
def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
"""
Decides if a user-defined custom prompt or default needs to be used
@ -5537,11 +5613,8 @@ def response_schema_prompt(model: str, response_schema: dict) -> str:
custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
if custom_prompt_details is not None:
return custom_prompt(
role_dict=custom_prompt_details["roles"],
initial_prompt_value=custom_prompt_details["initial_prompt_value"],
final_prompt_value=custom_prompt_details["final_prompt_value"],
messages=response_schema_as_message,
return render_custom_prompt_template(
template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
)
else:
return default_response_schema_prompt(response_schema=response_schema)
@ -5561,8 +5634,95 @@ def default_response_schema_prompt(response_schema: dict) -> str:
# Custom prompt template
def _mapping_or_none(value: object) -> object:
if value is None or issubclass(type(value), Mapping):
return value
verbose_logger.warning(
"Ignoring a custom prompt template roles value of type %s because it is not a mapping, "
"so those messages render without role markers",
type(value).__name__,
)
return None
class CustomPromptRole(BaseModel):
model_config = ConfigDict(frozen=True)
pre_message: str | None = None
post_message: str | None = None
class CustomPromptRoles(BaseModel):
model_config = ConfigDict(frozen=True, title="custom prompt template")
roles: Mapping[str, Annotated[CustomPromptRole | None, BeforeValidator(_mapping_or_none)]]
class CustomPromptTemplate(BaseModel):
model_config = ConfigDict(frozen=True, title="custom prompt template")
roles: Annotated[Mapping[object, object] | None, BeforeValidator(_mapping_or_none)] = None
initial_prompt_value: str | None = None
final_prompt_value: str | None = None
@model_validator(mode="after")
def _sets_a_rendered_field(self) -> "CustomPromptTemplate":
if self.roles is None and self.initial_prompt_value is None and self.final_prompt_value is None:
raise PydanticCustomError(
"empty_custom_prompt_template",
"it sets none of roles, initial_prompt_value, or final_prompt_value",
)
return self
def render_custom_prompt_template(
template: Mapping[str, object],
messages: Sequence[Mapping[str, object]],
model: str,
llm_provider: str,
) -> str:
try:
return _render_custom_prompt_template(template=template, messages=messages)
except ValidationError as error:
raise litellm.APIConnectionError(
message=f"The custom prompt template for {model} is invalid: {_describe_template_errors(error)}",
model=model,
llm_provider=llm_provider,
) from error
def _describe_template_errors(error: ValidationError) -> str:
return "; ".join(
_describe_template_error(detail) for detail in error.errors(include_url=False, include_input=False)
)
def _describe_template_error(detail: ErrorDetails) -> str:
location: Final = ".".join(str(part) for part in detail["loc"])
return f"{location}: {detail['msg']}" if location else detail["msg"]
def _render_custom_prompt_template(template: Mapping[str, object], messages: Sequence[Mapping[str, object]]) -> str:
validated: Final = CustomPromptTemplate.model_validate(template)
message_roles: Final = frozenset(message.get("role") for message in messages)
template_roles: Final = validated.roles or {}
used_roles: Final = CustomPromptRoles.model_validate(
{"roles": {role: entry for role, entry in template_roles.items() if role in message_roles}}
)
return custom_prompt(
role_dict={
role: {"pre_message": entry.pre_message or "", "post_message": entry.post_message or ""}
for role, entry in used_roles.roles.items()
if entry is not None
},
initial_prompt_value=validated.initial_prompt_value or "",
final_prompt_value=validated.final_prompt_value or "",
messages=list(messages),
)
def custom_prompt(
role_dict: dict,
role_dict: Mapping[str, Mapping[str, str]],
messages: list,
initial_prompt_value: str = "",
final_prompt_value: str = "",

15
litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
View file

@ -414,10 +414,11 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool:
def _renders(message: object, answered: frozenset[str]) -> bool:
"""Whether ``anthropic_messages_pt`` puts a block on the wire for this message.
A tool message always becomes a ``tool_result`` and a user message with string
content always becomes a text block (empty text gets a placeholder). A user list
renders only through parts of a type the converter emits; ``None``, an empty list,
and a list of other parts vanish. Assistant messages follow ``_assistant_renders``.
A tool message always becomes a ``tool_result``, and a user message with string or
missing content (``None``, an empty list) always puts its turn on the wire, an empty
turn getting a placeholder. A user list renders only through parts of a type the
converter emits, so a list of only other parts vanishes. Assistant messages follow
``_assistant_renders``.
"""
role: Final = message_field(message, "role")
if role in _TOOL_ROLES:
@ -425,8 +426,10 @@ def _renders(message: object, answered: frozenset[str]) -> bool:
if role == "assistant":
return _assistant_renders(message, answered)
content: Final = message_field(message, "content")
return isinstance(content, str) or any(
message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content)
return (
isinstance(content, str)
or not content
or any(message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content))
)

247
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,17 +1,20 @@
import asyncio
import json
import traceback
from collections.abc import Coroutine, Mapping, Sequence
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
from pydantic import TypeAdapter
from typing_extensions import ReadOnly
import litellm
from litellm._logging import redact_internal_details_from_client_message, verbose_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
@ -98,6 +101,16 @@ class ScopedWebSocket(Protocol):
def scope(self) -> _ASGIScope: ...
@runtime_checkable
class _ReplayScopeWebSocket(Protocol):
@property
def scope(self) -> MutableMapping[str, object]: ...
def client_websocket_scope(websocket: object) -> MutableMapping[str, object] | None:
return websocket.scope if isinstance(websocket, _ReplayScopeWebSocket) else None
class _ClientWebSocket(ScopedWebSocket, Protocol):
async def send_text(self, data: str) -> None: ...
async def receive_text(self) -> str: ...
@ -116,10 +129,87 @@ def _decode_json_object(payload: str) -> Mapping[str, object]:
return json.loads(payload)
_GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType(
{"type": "server_vad", "create_response": False}
)
_EMPTY_SESSION: Final[Mapping[str, object]] = MappingProxyType({})
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_GUARDRAIL_EVENT_ID_PREFIX: Final = "litellm_guardrail_"
def _json_object(value: object) -> Mapping[str, object] | None:
return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
return _json_object(container.get(key))
def _guardrail_event_id() -> str:
return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
def _error_event_id(event: Mapping[str, object]) -> str | None:
error: Final = _mapping_field(event, "error")
caused_by: Final = None if error is None else error.get("event_id")
return caused_by if isinstance(caused_by, str) else None
_TurnDetectionGate: TypeAlias = Callable[[object], Mapping[str, object]]
def _turn_detection_with_auto_response_off(turn_detection: object) -> Mapping[str, object]:
client_turn_detection: Final = _json_object(turn_detection)
if client_turn_detection is None:
return dict(_GUARDRAIL_TURN_DETECTION_GATE)
return {**client_turn_detection, "create_response": False}
def _nested_turn_detection_part(session: Mapping[str, object], gate: _TurnDetectionGate) -> Mapping[str, object]:
audio: Final = _mapping_field(session, "audio")
if audio is None:
return {}
audio_input: Final = _mapping_field(audio, "input")
if audio_input is None or "turn_detection" not in audio_input:
return {}
gated_input: Final = {
**audio_input,
"turn_detection": gate(audio_input["turn_detection"]),
}
return {"audio": {**audio, "input": gated_input}}
def _with_create_response_disabled(
session: Mapping[str, object], gate: _TurnDetectionGate
) -> Mapping[str, object] | None:
"""``session`` with every turn_detection the client sent unable to auto-respond, or None when it sent none."""
flat_part: Final = {"turn_detection": gate(session["turn_detection"])} if "turn_detection" in session else {}
nested_part: Final = _nested_turn_detection_part(session, gate)
if not flat_part and not nested_part:
return None
return {**session, **flat_part, **nested_part}
def _gate_session_update(
session: object, gate: _TurnDetectionGate, inject_gate_when_absent: bool
) -> Mapping[str, object] | None:
"""The session to forward under a transcript guardrail, or None when the frame needs no change."""
client_session: Final = _json_object(session)
if client_session is None:
return None
gated: Final = _with_create_response_disabled(client_session, gate)
if gated is not None or not inject_gate_when_absent:
return gated
return {**client_session, "turn_detection": gate(None)}
class RealtimeEventNormalizer(Protocol):
def should_drop(self, event: object) -> bool: ...
def normalize(self, event: dict) -> dict: ...
def patch_outgoing_session(self, session: dict) -> dict: ...
def gate_turn_detection(self, turn_detection: object, /) -> Mapping[str, object]: ...
@property
def deletes_conversation_items(self) -> bool: ...
class RealtimeUsageMeter(Protocol):
@ -154,6 +244,7 @@ class RealTimeStreaming:
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
):
self.websocket: _ClientWebSocket = websocket
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
self._logging_worker = logging_worker
@ -197,6 +288,8 @@ class RealTimeStreaming:
# Track whether we have already sent the guardrail turn-detection update
# that disables provider auto-response for transcription guardrails.
self._guardrail_turn_detection_update_sent: bool = False
self._session_gate_sent: bool = False
self._guardrail_cancel_event_ids: frozenset[str] = frozenset()
# Deferred Gemini Live setup: Pipecat may stream audio before session.update.
# Buffer client audio until the backend acknowledges setup (setupComplete).
self._backend_setup_complete: bool = provider_config is None or provider_config.requires_session_configuration()
@ -213,6 +306,12 @@ class RealTimeStreaming:
self._is_transcription_session: bool = force_transcription_model is not None
# Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
self._event_normalizer = event_normalizer
self._gate_turn_detection: Final[_TurnDetectionGate] = (
_turn_detection_with_auto_response_off if event_normalizer is None else event_normalizer.gate_turn_detection
)
self._deletes_conversation_items: Final = (
event_normalizer is None or event_normalizer.deletes_conversation_items
)
self._usage_meter = usage_meter
@property
@ -734,20 +833,37 @@ class RealTimeStreaming:
except (json.JSONDecodeError, TypeError):
return
def _make_turn_detection_update(self, turn_detection: Mapping[str, object]) -> dict[str, object]:
session: Final[dict[str, object]] = (
{"turn_detection": turn_detection}
if self._backend_uses_beta_protocol
else {"type": "realtime", "audio": {"input": {"turn_detection": turn_detection}}}
)
return {"type": "session.update", "session": session}
def _make_disable_auto_response_message(self) -> str:
"""Return a session.update that disables VAD auto-response."""
turn_detection: Final[dict[str, str | bool]] = {
"type": "server_vad",
"create_response": False,
}
if self._backend_uses_beta_protocol:
session: dict[str, object] = {"turn_detection": turn_detection}
else:
session = {
"type": "realtime",
"audio": {"input": {"turn_detection": turn_detection}},
}
return json.dumps({"type": "session.update", "session": session})
return json.dumps(self._make_turn_detection_update(self._gate_turn_detection(None)))
async def _send_guardrail_cancel(self) -> None:
event_id: Final = _guardrail_event_id()
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
await self._send_to_backend(json.dumps({"type": "conversation.item.delete", "item_id": item_id}))
def _is_guardrail_cancel_error(self, event: Mapping[str, object]) -> bool:
return _error_event_id(event) in self._guardrail_cancel_event_ids
async def _send_session_gate_once(self) -> None:
"""Gate the backend's default turn_detection ahead of the first client session.update, which overrides it."""
if self._session_gate_sent or not self._should_disable_vad_auto_response():
return
self._session_gate_sent = True
await self._send_to_backend(self._make_disable_auto_response_message())
async def _maybe_send_guardrail_turn_detection_update(self) -> None:
"""Disable provider auto-response once when transcription guardrails are enabled."""
@ -921,7 +1037,7 @@ class RealTimeStreaming:
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._send_to_backend(json.dumps({"type": "response.cancel"}))
await self._withdraw_blocked_turn(item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1051,10 +1167,7 @@ class RealTimeStreaming:
self.store_message(event_str)
self._capture_transcription_usage(event)
await self._send_event_to_client(event, event_str)
blocked = await self.run_realtime_guardrails(
cast(str, transcript),
item_id=cast(str | None, event.get("item_id")),
)
blocked = await self.run_realtime_guardrails(cast(str, transcript))
if not blocked and self._should_disable_vad_auto_response():
await self._send_to_backend(json.dumps({"type": "response.create"}))
continue
@ -1086,7 +1199,7 @@ class RealTimeStreaming:
if event_type == "session.created" and self._should_disable_vad_auto_response():
self.store_message(event_obj)
await self.websocket.send_text(self._event_to_client_json(event_obj))
await self._send_to_backend(self._make_disable_auto_response_message())
await self._send_session_gate_once()
return True
if event_type == "conversation.item.input_audio_transcription.completed":
@ -1140,6 +1253,10 @@ class RealTimeStreaming:
if self._usage_meter is not None:
self._usage_meter.observe_backend_event(event)
if event.get("type") == "error" and self._is_guardrail_cancel_error(event):
verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
continue
if await self._handle_raw_backend_message(event, raw_response):
continue
@ -1346,6 +1463,13 @@ class RealTimeStreaming:
return item
async def _receive_client_message(self) -> str | None:
refused_opening_message: Final = (
None
if self._replay_scope is None
else self._replay_scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
)
if isinstance(refused_opening_message, str):
return refused_opening_message
try:
return await self.websocket.receive_text()
except Exception as e: # noqa: BLE001 # whatever the client socket raises, the client is gone
@ -1472,19 +1596,21 @@ class RealTimeStreaming:
and self.session_configuration_request is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
):
session: Mapping[str, object] | None = msg_obj.setdefault("session", {})
if isinstance(session, dict):
existing_td = session.get("turn_detection")
if not isinstance(existing_td, dict):
existing_td = {}
existing_td["create_response"] = False
session["turn_detection"] = existing_td
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
and (
gated_first := _gate_session_update(
client_event.get("session", _EMPTY_SESSION),
self._gate_turn_detection,
inject_gate_when_absent=True,
)
)
is not None
):
msg_obj["session"] = gated_first
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
)
## GUARDRAIL: Force ``create_response`` to False in any
# client-provided ``turn_detection`` so a later
@ -1492,52 +1618,25 @@ class RealTimeStreaming:
# and bypass the transcription guardrail after the
# initial disable. Covers both the flat beta key and the
# nested GA ``audio.input.turn_detection`` shape, since
# the GA remap below also accepts either form. Skipped
# when the injection block above already ran for this
# message, to avoid redundant double-serialization.
# the GA remap below also accepts either form. A later
# update with no turn_detection is forwarded as is: OpenAI
# keeps the earlier disable for an unset field, and xAI
# falls back to its manual default, which never auto-responds.
if (
msg_type == "session.update"
and not guardrail_turn_detection_injected
and self._should_disable_vad_auto_response()
and (
gated_later := _gate_session_update(
client_event.get("session"),
self._gate_turn_detection,
inject_gate_when_absent=False,
)
)
is not None
):
session = client_event.get("session")
if isinstance(session, dict):
td_overridden = False
flat_td = session.get("turn_detection")
flat_td_present = flat_td is not None
if flat_td_present:
if not isinstance(flat_td, dict):
flat_td = {}
if flat_td.get("create_response") is not False:
flat_td["create_response"] = False
session["turn_detection"] = flat_td
td_overridden = True
nested_td_present = False
audio = session.get("audio")
if isinstance(audio, dict):
audio_input = audio.get("input")
if isinstance(audio_input, dict):
nested_td = audio_input.get("turn_detection")
if nested_td is not None:
nested_td_present = True
if not isinstance(nested_td, dict):
nested_td = {}
if nested_td.get("create_response") is not False:
nested_td["create_response"] = False
audio_input["turn_detection"] = nested_td
td_overridden = True
# Symmetric with the first-update injection block:
# if the client omitted turn_detection entirely on
# a subsequent session.update, still inject the
# ``create_response: False`` override so the
# transcription guardrail cannot be re-enabled by
# any downstream merge that drops the original
# disable.
if not flat_td_present and not nested_td_present:
session["turn_detection"] = {"create_response": False}
td_overridden = True
if td_overridden:
message = json.dumps(msg_obj)
msg_obj["session"] = gated_later
message = json.dumps(msg_obj)
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
@ -1590,6 +1689,8 @@ class RealTimeStreaming:
# would permanently disable the injection if ``_send_to_backend``
# raised — neither this loop nor
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
sent = await self._send_to_backend(message)
if guardrail_turn_detection_injected and sent:
self._guardrail_turn_detection_update_sent = True

96
litellm/litellm_core_utils/streaming_handler.py
View file

@ -242,6 +242,7 @@ class CustomStreamWrapper:
self.completion_stream = completion_stream
self.sent_first_chunk = False
self.sent_last_chunk = False
self._failure_logging: asyncio.Task[None] | None = None
self._closed = False
self._stream_created_time: float = time.time()
@ -269,6 +270,7 @@ class CustomStreamWrapper:
self.holding_chunk = ""
self.complete_response = ""
self.response_uptil_now = ""
self._emitted_disqualifying_content = False
_model_info: Final[dict] = litellm_params.model_info or {}
_api_base: Final = get_api_base(
@ -1126,31 +1128,24 @@ class CustomStreamWrapper:
"""
if self.merge_reasoning_content_in_choices is True:
reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None)
if reasoning_content:
if self.sent_first_thinking_block is False:
# Ensure content is not None before concatenation
if model_response.choices[0].delta.content is None:
model_response.choices[0].delta.content = ""
model_response.choices[0].delta.content += "<think>" + reasoning_content
self.sent_first_thinking_block = True
elif (
self.sent_first_thinking_block is True
and hasattr(model_response.choices[0].delta, "reasoning_content")
and model_response.choices[0].delta.reasoning_content
):
model_response.choices[0].delta.content = reasoning_content
elif (
self.sent_first_thinking_block is True
and not self.sent_last_thinking_block
and model_response.choices[0].delta.content
):
model_response.choices[0].delta.content = "</think>" + (model_response.choices[0].delta.content or "")
self.sent_last_thinking_block = True
if self.merge_reasoning_content_in_choices is not True:
return
delta: Final = model_response.choices[0].delta
reasoning_content: Final = getattr(delta, "reasoning_content", None)
content: Final = delta.content or ""
if reasoning_content:
self._emitted_disqualifying_content = True
closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
opening: Final = "" if self.sent_first_thinking_block else "<think>"
delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
self.sent_first_thinking_block = True
self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
delta.content = "</think>" + content
self.sent_last_thinking_block = True
if hasattr(model_response.choices[0].delta, "reasoning_content"):
del model_response.choices[0].delta.reasoning_content
if hasattr(delta, "reasoning_content"):
del delta.reasoning_content
def _dispatch_provider_chunk(
self,
@ -1865,9 +1860,7 @@ class CustomStreamWrapper:
if response.choices:
choice = response.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# HANDLE STREAM OPTIONS
self.chunks.append(response)
@ -2084,9 +2077,7 @@ class CustomStreamWrapper:
if processed_chunk.choices:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# Add mcp_list_tools to first chunk if present
if not self.sent_first_chunk and processed_chunk.choices:
@ -2156,9 +2147,7 @@ class CustomStreamWrapper:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# RETURN RESULT
self.chunks.append(processed_chunk)
@ -2172,7 +2161,7 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
await self.aclose()
@ -2305,11 +2294,17 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
self._handle_stream_fallback_error(e)
async def wait_for_failure_logging(self, timeout: float) -> None:
"""Let this stream's failure log read its own deployment before a fallback reuses the logging object."""
if self._failure_logging is None:
return
await asyncio.wait((self._failure_logging,), timeout=timeout)
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
@ -2346,6 +2341,35 @@ class CustomStreamWrapper:
recover_error,
)
# Delta fields a text-only prefill continuation cannot carry, so a stream
# that emitted any of them is not eligible for mid-stream continuation.
_CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = (
"tool_calls",
"function_call",
"reasoning_content",
"thinking_blocks",
"reasoning_items",
"audio",
"images",
"annotations",
)
@classmethod
def _delta_disqualifies_continuation(cls, delta: object) -> bool:
get: Final = getattr(delta, "get", None)
if not callable(get):
return False
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
def _accumulate_streamed_delta(self, delta: object) -> None:
# Shared by the sync, async, and non-aiohttp iteration sites so answer
# text and the disqualifying-content latch stay in step across all three.
get: Final = getattr(delta, "get", None)
content: Final = get("content") if callable(get) else None
self.response_uptil_now += content if isinstance(content, str) else ""
if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
self._emitted_disqualifying_content = True
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
@ -2377,6 +2401,7 @@ class CustomStreamWrapper:
original_exception=e.original_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content,
)
# Map to OpenAI exception format. Some providers' mappers (e.g.
@ -2437,6 +2462,7 @@ class CustomStreamWrapper:
original_exception=mapped_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content,
)
@staticmethod

164
litellm/litellm_core_utils/tool_fields.py Normal file
View file

@ -0,0 +1,164 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
ROUTES_WITH_TOOLS: Final = frozenset(
{
"acompletion",
"aresponses",
"acompact_responses",
"anthropic_messages",
"agenerate_content",
"agenerate_content_stream",
"acreate_interaction",
}
)
TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
_TOOLS: Final = TypeAdapter(list[object])
_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
def _schema_as_read(value: object) -> object:
if not value:
return None
if not isinstance(value, str):
return value
try:
return _JSON_OBJECT.validate_python(value)
except ValidationError:
return value
_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
class _ToolFunction(TypedDict, total=False):
parameters: ReadOnly[_Schema]
class _ToolEntry(TypedDict, total=False):
function: ReadOnly[_ToolFunction | None]
parameters: ReadOnly[_Schema]
input_schema: ReadOnly[_Schema]
vector_store_ids: ReadOnly[list[str] | None]
_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
@dataclass(frozen=True, slots=True)
class DropPolicy:
drop_params: bool
additional_drop_params: frozenset[str]
def drops(self, field: str) -> bool:
return self.drop_params or field in self.additional_drop_params
@dataclass(frozen=True, slots=True)
class MalformedField:
field: str
message: str
dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
def _field_names(value: object) -> frozenset[str]:
try:
return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
except ValidationError:
return frozenset()
def drop_policy(params: Mapping[str, object]) -> DropPolicy:
return DropPolicy(
drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
additional_drop_params=_field_names(params.get("additional_drop_params")),
)
def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
error: Final = e.errors(include_url=False)[0]
return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
def _tool_entry_error(index: int, entry: object) -> str | None:
try:
_TOOL_ENTRY.validate_python(entry, strict=True)
except ValidationError as e:
return first_validation_error(e, prefix=("tools", index))
return None
def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
tools: Final = data.get("tools")
if tools is None:
return None
try:
entries: Final = _TOOLS.validate_python(tools)
except ValidationError as e:
return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
first_error: Final = next((error for error in entry_errors if error is not None), None)
if first_error is None:
return None
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
return MalformedField(
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
)
def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name != "web_search_options"}
def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
try:
_WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
except ValidationError as e:
return MalformedField(
"web_search_options",
first_validation_error(e, prefix=("web_search_options",)),
_without_web_search_options,
)
return None
def resolve_tool_fields(
data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
) -> Mapping[str, object] | MalformedField:
checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
malformed: Final = tuple(field for field in checked if field is not None)
if not malformed:
return data
policies: Final = candidates()
rejected: Final = next(
(field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
)
if rejected is not None:
return rejected
return reduce(
lambda resolved, field: field.dropped(resolved),
(field for field in malformed if all(policy.drops(field.field) for policy in policies)),
data,
)
def resolve_deployment_tool_fields(
kwargs: Mapping[str, object], route: object
) -> Mapping[str, object] | MalformedField:
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
return kwargs
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))

5
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
scoped_structured_message_indices,
@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation):
images=(),
)
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract every tool name in an Anthropic messages request: tools[].name, plus
tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
@classmethod
def _extract_input_text_and_images(

114
litellm/llms/anthropic/chat/transformation.py
View file

@ -1,3 +1,4 @@
import copy
import json
import re
import time
@ -24,7 +25,18 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
is_object_schema,
sanitize_input_schema_for_anthropic,
schema_fragment_ref,
schema_grammar_branch_count,
schema_has_lookaround_pattern,
schema_has_open_object,
schema_has_unresolvable_ref,
schema_optional_property_count,
schema_string_formats,
schema_union_property_count,
unpack_defs,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
RemoteMedia,
@ -43,7 +55,7 @@ from litellm.llms.base_llm.base_utils import (
type_to_response_format_param,
unwrap_response_format_envelope,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
@ -150,6 +162,12 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
_DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
NATIVE_OUTPUT_MAX_UNION_PARAMETERS: Final = 16
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24
NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048
def _tool_type(tool: object) -> str | None:
@ -753,7 +771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
grammar Anthropic compiles the schema into, so the API rejects them with a
400 ``invalid_request_error`` (e.g. "output_format.schema: For 'array' type,
property 'uniqueItems' is not supported"):
- maxItems/minItems/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- maxItems/minItems above 1/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- minimum/maximum/exclusiveMinimum/exclusiveMaximum/multipleOf: numeric constraints
- minLength/maxLength: string constraints
- minProperties/maxProperties/patternProperties/propertyNames: object constraints
@ -810,7 +828,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"else": "conditional (else): {}",
"not": "must not match: {}",
}
unsupported_fields: Final = set(constraint_labels)
min_items: Final = schema.get("minItems")
enforced_min_items: Final = (
isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1)
)
unsupported_fields: Final = set(constraint_labels) - ({"minItems"} if enforced_min_items else set())
# Build description additions from removed constraints. Iterating
# constraint_labels (not the set) keeps the note order deterministic across
@ -818,7 +840,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# PYTHONHASHSEED and stay cache-friendly.
constraint_descriptions: Final[list] = []
for field, label in constraint_labels.items():
if field not in schema:
if field not in schema or field not in unsupported_fields:
continue
value = schema[field]
# A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no
@ -853,7 +875,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Already handled above
continue
if key in ("properties", "$defs") and isinstance(value, dict):
if key in ("properties", "$defs", "definitions") and isinstance(value, dict):
nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()}
elif key == "items" and isinstance(value, dict):
@ -865,6 +887,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
]
elif key == "allOf" and isinstance(value, list):
result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value]
elif key == "$ref" and isinstance(value, str):
result[key] = schema_fragment_ref(value)
elif key == "oneOf" and isinstance(value, list):
# Anthropic rejects oneOf ("Schema type 'oneOf' is not supported");
# the Anthropic SDK rewrites it to anyOf, so do the same.
@ -876,7 +900,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Anthropic requires additionalProperties=false for object schemas
# See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs
if result.get("type") == "object":
if is_object_schema(result):
result["additionalProperties"] = False
return result
@ -1628,32 +1652,55 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return json_schema
@staticmethod
def _defs_unpacked(json_schema: Mapping[str, object]) -> Mapping[str, object]:
unpacked: Final = copy.deepcopy(dict(json_schema))
defs: Final = unpacked.pop("$defs", unpacked.pop("definitions", None))
if isinstance(defs, dict) and defs:
unpack_defs(unpacked, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
return unpacked
@staticmethod
def native_output_schema(json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
if (
schema_has_unresolvable_ref(json_schema)
or schema_has_open_object(json_schema)
or schema_has_lookaround_pattern(json_schema)
or not schema_string_formats(json_schema) <= _NATIVE_OUTPUT_STRING_FORMATS
or schema_union_property_count(json_schema) > NATIVE_OUTPUT_MAX_UNION_PARAMETERS
or schema_optional_property_count(json_schema) > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS
or schema_grammar_branch_count(json_schema) > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES
):
return None
return AnthropicConfig.filter_anthropic_output_schema(dict(json_schema))
def uses_native_structured_output_for_schema(
self,
model: str,
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
open_object_takes_forced_json_tool: Final = (
json_schema is not None
and schema_has_open_object(json_schema)
and not self.is_thinking_requested(request_params)
and not self.is_thinking_enabled(dict(request_params))
and not request_params.get("tools")
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
)
return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema)))
def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None:
json_schema: dict | None = self._extract_json_schema_from_response_format(value)
json_schema: Final = self._extract_json_schema_from_response_format(value)
if json_schema is None:
return None
# Resolve $ref/$defs before filtering — Anthropic doesn't support
# external schema references (e.g., /$defs/CalendarEvent).
import copy
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
unpack_defs,
)
json_schema = copy.deepcopy(json_schema)
defs: Final = json_schema.pop("$defs", json_schema.pop("definitions", {}))
if defs:
unpack_defs(json_schema, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
# Filter out unsupported fields for Anthropic's output_format API
filtered_schema: Final = self.filter_anthropic_output_schema(json_schema)
return AnthropicOutputSchema(
type="json_schema",
schema=filtered_schema,
)
native_schema: Final = self.native_output_format_schema(json_schema)
if native_schema is None:
return None
return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema))
def map_response_format_to_anthropic_tool(
self, value: dict | None, optional_params: dict, is_thinking_enabled: bool
@ -1820,7 +1867,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_key=param,
)
elif param == "response_format" and isinstance(value, dict):
if self.uses_native_structured_output(model):
if self.uses_native_structured_output_for_schema(
model,
self._extract_json_schema_from_response_format(value),
REQUEST_PARAMS.validate_python(non_default_params),
):
_output_format = self.map_response_format_to_anthropic_output_format(value)
if _output_format is not None:
optional_params["output_format"] = _output_format
@ -2979,6 +3030,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
completion_response["stop_reason"] = "stop"
if refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks
json_mode_message.reasoning_content = reasoning_content
_message = json_mode_message
model_response.choices[0].message = _message

9
litellm/llms/anthropic/completion/transformation.py
View file

@ -14,8 +14,8 @@ import httpx
import litellm
from litellm.constants import DEFAULT_MAX_TOKENS
from litellm.litellm_core_utils.prompt_templates.factory import (
custom_prompt,
prompt_factory,
render_custom_prompt_template,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import (
@ -244,11 +244,8 @@ class AnthropicTextConfig(BaseConfig):
if model in custom_prompt_dict:
# check if the model has a registered custom prompt
model_prompt_details: Final = custom_prompt_dict[model]
prompt = custom_prompt(
role_dict=model_prompt_details["roles"],
initial_prompt_value=model_prompt_details["initial_prompt_value"],
final_prompt_value=model_prompt_details["final_prompt_value"],
messages=messages,
prompt = render_custom_prompt_template(
template=model_prompt_details, messages=messages, model=model, llm_provider="anthropic"
)
else:
prompt = prompt_factory(model=model, messages=messages, custom_llm_provider="anthropic")

20
litellm/llms/anthropic/count_tokens/token_counter.py
View file

@ -6,11 +6,12 @@ from typing import Any, Final
from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import AuthenticationError
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler()
@ -36,6 +37,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
system: JsonValue = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Anthropic's CountTokens API.
@ -61,14 +63,24 @@ class AnthropicTokenCounter(BaseTokenCounter):
try:
auth_header: Final = await AnthropicModelInfo.aget_auth_header(
api_key=litellm_params.get("api_key"),
api_key=litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key,
api_base=api_base,
litellm_params=litellm_params,
allow_workload_identity=True,
)
if auth_header is None:
verbose_logger.warning("No Anthropic credential found for token counting")
return None
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="anthropic_api",
error=True,
error_message=(
f"No Anthropic API key is set for model {request_model or model_to_use}, so tokens can't be "
"counted. Set api_key on the deployment or ANTHROPIC_API_KEY in the proxy environment."
),
status_code=401,
)
result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
model=model_to_use,

93
litellm/llms/anthropic/pass_through/adapters/handler.py
View file

@ -1,4 +1,5 @@
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Final,
@ -62,6 +63,12 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
custom_llm_provider: str
@dataclass(frozen=True, slots=True)
class _ChatTargetReasoning:
effort: str
summary: str | None = None
def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool:
"""Return True when any message carries a ``compaction`` content block."""
for msg in messages:
@ -132,7 +139,7 @@ def request_safeguards_evaluator(
)
async def _prepare_context_managed_request(
async def prepare_context_managed_request(
*,
model: str,
messages: _AnthropicMessages,
@ -145,6 +152,8 @@ async def _prepare_context_managed_request(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
drop_invalid_edits: bool = False,
) -> PolyfillResult | None:
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
@ -186,6 +195,8 @@ async def _prepare_context_managed_request(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
drop_invalid_edits=drop_invalid_edits,
)
if polyfill_result is not None:
@ -259,16 +270,14 @@ def _spec_has_non_compact_edits(
return any(isinstance(edit.get("type"), str) and edit.get("type") != COMPACT_EDIT_TYPE for edit in edits)
def _context_management_explicitly_dropped(additional_drop_params: list[str] | None) -> bool:
def context_management_explicitly_dropped(additional_drop_params: object) -> bool:
"""True when the caller opted out of context_management via ``additional_drop_params``.
``drop_params`` deliberately does NOT gate the polyfill: ``context_management``
is a LiteLLM-supported param (native on Anthropic, polyfilled elsewhere), and
``drop_params`` only exists to drop genuinely unsupported params.
"""
if not isinstance(additional_drop_params, list):
return False
return "context_management" in additional_drop_params
return isinstance(additional_drop_params, list) and "context_management" in additional_drop_params
def _normalize_spec_edits(
@ -284,7 +293,7 @@ def _normalize_spec_edits(
if not context_management_spec:
return None
if _context_management_explicitly_dropped(additional_drop_params):
if context_management_explicitly_dropped(additional_drop_params):
return None
from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
@ -310,20 +319,23 @@ async def _run_polyfill_if_enabled(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
drop_invalid_edits: bool = False,
) -> PolyfillResult | None:
"""Run the async context_management polyfill if a spec is present.
Returns ``None`` when the spec is empty or ``context_management`` is
listed in ``additional_drop_params`` (the explicit opt-out; ``drop_params``
does not disable the polyfill because context_management is a supported
does not disable a valid spec because context_management is a supported
param). Raises ``AnthropicContextManagementError`` so the /v1/messages
endpoint can emit an Anthropic-format 400. All other exceptions are
endpoint can emit an Anthropic-format 400, unless ``drop_invalid_edits``
(``drop_params``) drops the invalid spec. All other exceptions are
best-effort swallowed (matches v0 behavior).
"""
if not context_management_spec:
return None
if _context_management_explicitly_dropped(additional_drop_params):
if context_management_explicitly_dropped(additional_drop_params):
return None
try:
@ -338,13 +350,17 @@ async def _run_polyfill_if_enabled(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
)
except AnthropicContextManagementError:
if drop_invalid_edits:
return None
# Surface validation errors so the endpoint can emit an Anthropic-format
# 400. Other exception types fall into the best-effort branch below.
raise
except Exception as e:
verbose_logger.exception("context_management polyfill: skipping edits due to error: %s", e)
log_failure: Final = verbose_logger.debug if count_only else verbose_logger.exception
log_failure("context_management polyfill: skipping edits due to error: %s", e)
# Best-effort swallow is only safe for compact-only specs, where the
# caller's compaction-block-slicing safety net produces a correct
# (if degraded) result. When the spec also requested non-compact
@ -441,15 +457,33 @@ class LiteLLMMessagesToCompletionTransformationHandler:
completion_kwargs["reasoning_effort"] = updated_reasoning_effort
@staticmethod
def _plain_effort_for_chat_target(
def _requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None:
if thinking.get("type") == "disabled":
return None
summary: Final = thinking.get("summary")
if isinstance(summary, str):
return summary
display: Final = thinking.get("display")
return "auto" if display is not None and display != "omitted" else None
@staticmethod
def _chat_target_reasoning(
completion_kwargs: _CompletionKwargs,
*,
thinking: Mapping[str, object] | None,
) -> str | None:
reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
if not thinking or not isinstance(reasoning_effort, dict) or "summary" not in reasoning_effort:
) -> _ChatTargetReasoning | None:
if not thinking:
return None
effort: Final = reasoning_effort.get("effort")
reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
wraps_summary: Final = isinstance(reasoning_effort, dict) and "summary" in reasoning_effort
requested_summary: Final = LiteLLMMessagesToCompletionTransformationHandler._requested_reasoning_summary(
thinking
)
if not wraps_summary and requested_summary is None:
return None
effort: Final[object] = (
reasoning_effort.get("effort") if isinstance(reasoning_effort, dict) else reasoning_effort
)
model: Final = completion_kwargs.get("model")
if not isinstance(effort, str) or not isinstance(model, str) or not model:
return None
@ -467,7 +501,12 @@ class LiteLLMMessagesToCompletionTransformationHandler:
return None
if resolved_provider == "litellm_proxy":
return None
from litellm.responses.bridge_check import responses_api_bridge_check
from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check
if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider):
return _ChatTargetReasoning(effort=effort, summary=requested_summary)
if not wraps_summary or not isinstance(reasoning_effort, dict):
return None
web_search_options: Final = completion_kwargs.get("web_search_options")
tools: Final = completion_kwargs.get("tools")
@ -481,7 +520,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
reasoning_effort=reasoning_effort,
api_base=resolved_api_base,
)
return None if model_info.get("mode") == "responses" else effort
return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort)
@staticmethod
def _normalize_reasoning_effort(
@ -647,12 +686,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
thinking=thinking,
)
plain_effort: Final = LiteLLMMessagesToCompletionTransformationHandler._plain_effort_for_chat_target(
chat_reasoning: Final = LiteLLMMessagesToCompletionTransformationHandler._chat_target_reasoning(
completion_kwargs,
thinking=thinking,
)
if plain_effort is not None:
completion_kwargs["reasoning_effort"] = plain_effort
if chat_reasoning is not None:
completion_kwargs["reasoning_effort"] = chat_reasoning.effort
if chat_reasoning is not None and chat_reasoning.summary is not None:
completion_kwargs["reasoning_summary"] = chat_reasoning.summary
return completion_kwargs, tool_name_mapping
@ -677,6 +718,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
"""Handle non-Anthropic models asynchronously using the adapter"""
context_management: Final = kwargs.pop("context_management", None)
additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
drop_invalid_edits: Final = litellm.drop_params is True or (
"drop_params" in kwargs and kwargs["drop_params"] is True
)
litellm_router: Final[Router | None] = _router_or_proxy_fallback(kwargs.pop("litellm_router", None))
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
@ -684,7 +728,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs)
sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs)
polyfill_result: Final = await _prepare_context_managed_request(
polyfill_result: Final = await prepare_context_managed_request(
model=model,
messages=messages,
tools=tools,
@ -696,6 +740,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=sub_call_trace_id,
drop_invalid_edits=drop_invalid_edits,
)
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages
@ -809,6 +854,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
# bridge to it via ``run_async_function``.
context_management: Final = kwargs.pop("context_management", None)
additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
drop_invalid_edits: Final = litellm.drop_params is True or (
"drop_params" in kwargs and kwargs["drop_params"] is True
)
# Deliberately do NOT auto-attach the proxy ``llm_router`` here:
# ``run_async_function`` spawns a new event loop in a worker thread
# to bridge to the async dispatcher, but the proxy router's httpx
@ -834,7 +882,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs)
polyfill_result = run_async_function(
_prepare_context_managed_request,
prepare_context_managed_request,
model=model,
messages=messages,
tools=tools,
@ -846,6 +894,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=proxy_sub_call_logging_kwargs(request_kwargs),
parent_trace_id=proxy_sub_call_trace_id(request_kwargs),
drop_invalid_edits=drop_invalid_edits,
)
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages

75
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py
View file

@ -24,6 +24,7 @@ from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
@ -39,6 +40,7 @@ from litellm.types.llms.anthropic import (
UsageDelta,
UsageIteration,
)
from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDeltaToolCall, Delta, Function
if TYPE_CHECKING:
@ -50,6 +52,7 @@ if TYPE_CHECKING:
_STREAMING_DELTA_TYPES: Final = frozenset(get_args(StreamingContentBlockDeltaType))
_SSE_EVENT: Final = TypeAdapter(Mapping[str, object])
_THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock)
class _UsageDeltaWithIterations(UsageDelta, total=False):
@ -106,6 +109,14 @@ def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
assert_never(delta_type)
def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None:
delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
first_block: Final = as_str_mapping(blocks[0]) if blocks else None
index: Final = first_block.get("index") if first_block is not None else None
return index if isinstance(index, int) else None
def _optional_str_attr(obj: object, name: str) -> str | None:
value: Final = _optional_attr(obj, name)
return value if isinstance(value, str) else None
@ -373,13 +384,44 @@ class _CombinedChunkSplitter:
finish_delta.thinking_blocks = None
return [content_chunk, finish_chunk]
@staticmethod
def _with_thinking_block(
chunk: "ModelResponseStream",
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
*,
keep_role: bool,
) -> "ModelResponseStream":
piece: Final = copy.deepcopy(chunk)
role: Final = getattr(piece.choices[0].delta, "role", None) if keep_role else None
thinking: Final = block.get("thinking") if block["type"] == "thinking" else None
piece.choices[0].delta = Delta(role=role, reasoning_content=thinking or None, thinking_blocks=[block])
return piece
@staticmethod
def _split_thinking_blocks(chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
"""Anthropic streams one content block per thinking block, so a delta carrying several is split per block."""
choices: Final = _optional_attr_sequence(chunk, "choices")
if len(choices) != 1:
return (chunk,)
delta: Final = _optional_attr(choices[0], "delta")
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
if len(blocks) < 2 or _optional_attr(delta, "content") or _optional_attr(delta, "tool_calls"):
return (chunk,)
return tuple(
_CombinedChunkSplitter._with_thinking_block(
chunk, _THINKING_BLOCK.validate_python(block), keep_role=position == 0
)
for position, block in enumerate(blocks)
)
def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
split_chunks: Final = tuple(
split_chunk
for combined_chunk in self._split(chunk)
for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits
)
return tuple(itertools.chain.from_iterable(self._route_tool_calls(split_chunk) for split_chunk in split_chunks))
block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))
return tuple(itertools.chain.from_iterable(map(self._route_tool_calls, block_chunks)))
def __iter__(self) -> "Iterator[ModelResponseStream]":
return self
@ -458,7 +500,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
sent_first_chunk: bool = False
sent_content_block_start: bool = False
sent_content_block_finish: bool = False
current_content_block_type: Literal["text", "tool_use", "thinking"] = "text"
current_content_block_type: Literal["text", "tool_use", "thinking", "redacted_thinking"] = "text"
current_thinking_block_index: int | None = None
sent_last_message: bool = False
holding_chunk: ContentBlockDelta | None = None
holding_stop_reason_chunk: MessageBlockDelta | None = None
@ -1254,13 +1297,14 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
Also gates re-emission after a block transition: when an upstream
chunk both *triggers* a new content block (its type differs from the
active block) and *carries* delta content, that content belongs to
the new block. The synthesized ``content_block_start`` only ever
carries an empty body — see
``_translate_streaming_openai_chunk_to_anthropic_content_block``,
which returns an empty ``TextBlock``/``ToolUseBlock``/thinking block —
so the trigger chunk's delta must be re-queued or the first token of
the new block (the first non-empty text/thinking delta, or bundled
tool arguments) is silently dropped.
the new block. The synthesized ``content_block_start`` from
``_translate_streaming_openai_chunk_to_anthropic_content_block``
carries an empty text, tool-use or unsigned thinking body, and only a
signed thinking block's text rides on the start because its delta is
a ``signature_delta``. So the trigger chunk's delta must be re-queued
or the first token of the new block (the first non-empty text or
unsigned thinking delta, or bundled tool arguments) is silently
dropped.
Delta types outside ``StreamingContentBlockDeltaType`` — the closed
set the translate layer can produce — are treated as empty. The
@ -1352,11 +1396,24 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name)
tool_block["name"] = original_name
thinking_block_index: Final = _first_thinking_block_index(chunk)
previous_thinking_block_index: Final = self.current_thinking_block_index
if thinking_block_index is not None or block_type != self.current_content_block_type:
self.current_thinking_block_index = thinking_block_index
if block_type != self.current_content_block_type:
self.current_content_block_type = block_type
self.current_content_block_start = content_block_start
return True
if (
thinking_block_index is not None
and previous_thinking_block_index is not None
and thinking_block_index != previous_thinking_block_index
):
self.current_content_block_start = content_block_start
return True
# For parallel tool calls, we'll necessarily have a new content block
# if we get a function name since it signals a new tool call
if block_type == "tool_use":

8
litellm/llms/anthropic/pass_through/adapters/transformation.py
View file

@ -1627,7 +1627,7 @@ class LiteLLMAnthropicMessagesAdapter:
def translate_streaming_openai_chunk_to_anthropic_content_block(
self, choices: list[OpenAIStreamingChoice | StreamingChoices]
) -> tuple[
Literal["text", "tool_use", "thinking"],
Literal["text", "tool_use", "thinking", "redacted_thinking"],
"ContentBlockContentBlockDict",
]:
from litellm._uuid import uuid
@ -1664,6 +1664,10 @@ class LiteLLMAnthropicMessagesAdapter:
thinking_blocks = choice.delta.thinking_blocks or []
if len(thinking_blocks) > 0:
thinking_block = thinking_blocks[0]
if thinking_block["type"] == "redacted_thinking":
return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
type="redacted_thinking", data=thinking_block.get("data") or ""
)
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""
signature = thinking_block.get("signature") or ""
@ -1672,7 +1676,7 @@ class LiteLLMAnthropicMessagesAdapter:
assert isinstance(signature, str)
return "thinking", ChatCompletionThinkingBlock(
type="thinking", thinking=thinking, signature=signature
type="thinking", thinking=thinking if signature else "", signature=signature
)
# OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning
# parsers) populate ``reasoning_content`` without ``thinking_blocks``.

2
litellm/llms/anthropic/pass_through/context_management/dispatcher.py
View file

@ -85,6 +85,7 @@ async def apply_context_management(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: "ProxySubCallLoggingKwargs | None" = None,
parent_trace_id: str | None = None,
count_only: bool = False,
) -> PolyfillResult:
"""Run edits in order; return a single ``PolyfillResult``.
@ -128,6 +129,7 @@ async def apply_context_management(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
)
if editor_is_async
else await asyncify(editor)(

3
litellm/llms/anthropic/pass_through/context_management/editors/compact.py
View file

@ -642,6 +642,7 @@ async def apply_compact_20260112(
user_api_key_auth: Optional["UserAPIKeyAuth"] = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
) -> PolyfillResult:
"""Apply ``compact_20260112``; return a ``PolyfillResult``.
@ -682,7 +683,7 @@ async def apply_compact_20260112(
# Opt-in gate: no summary model configured → no-op (but still return the
# Phase A-sliced/stripped messages so compaction blocks don't leak).
summary_model: Final = _read_summary_model_setting()
summary_model: Final = None if count_only else _read_summary_model_setting()
if summary_model is None:
applied["error"] = "summary_model_not_configured"
# Slice-only forwarding: ``augmented_system`` already carries any prior

17
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -2,6 +2,7 @@ import asyncio
import json
import re
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
from contextvars import ContextVar
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None:
await stream.aclose()
_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False)
async def aclose_superseded_attempt(stream: object) -> None:
"""Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure."""
token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True)
try:
await aclose_if_supported(stream)
finally:
_CLOSING_SUPERSEDED_ATTEMPT.reset(token)
def closing_superseded_attempt() -> bool:
return _CLOSING_SUPERSEDED_ATTEMPT.get()
_RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])

10
litellm/llms/anthropic/pass_through/responses_adapters/handler.py
View file

@ -43,6 +43,12 @@ _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
logging_obj.record_bridged_responses_api_response(result)
def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]:
"""The litellm-specific kwargs forwarded verbatim onto the Responses API request."""
return extra_kwargs or {}
@ -301,7 +307,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
_record_bridged_response(responses_kwargs, result)
return served
@staticmethod
def anthropic_messages_handler(

1
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
status_code=400,
message=(
"Azure OpenAI does not support reasoning_effort='none' for this model. "
"Supported values are: 'low', 'medium', and 'high'. "
"To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
"Issue: https://github.com/BerriAI/litellm/issues/16704"

10
litellm/llms/azure/common_utils.py
View file

@ -2,6 +2,7 @@ import asyncio
import hashlib
import json
import os
import re
from collections.abc import Callable, Mapping
from functools import lru_cache
from types import MappingProxyType
@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.utils import add_path_to_api_base
azure_ad_cache: Final = DualCache()
_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)")
class _AzureAdTokenJson(TypedDict, total=False):
@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM):
) -> dict[str, str]: # mutable-ok: mirrors override contract
return cls._base_validate_azure_environment(headers, litellm_params)
@staticmethod
def resolve_azure_api_base(api_base: str | None) -> str | None:
return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@staticmethod
def _get_base_azure_url(
api_base: str | None,
@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM):
default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
"""
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_base = BaseAzureLLM.resolve_azure_api_base(api_base)
if api_base is None:
raise ValueError(
f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url: Final = httpx.URL(new_url)
new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1)))
# Use the new query_params dictionary
final_url: Final = httpx.URL(new_url).copy_with(params=query_params)

1
litellm/llms/azure/containers/transformation.py
View file

@ -28,6 +28,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return BaseAzureLLM.base_validate_azure_environment(
headers=headers,

12
litellm/llms/azure/fine_tuning/handler.py
View file

@ -1,4 +1,4 @@
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import Final, cast
import httpx
@ -66,6 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
self._ensure_training_type(create_fine_tuning_job_data)
@ -78,6 +79,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -109,6 +111,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -119,6 +122,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -149,6 +153,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -159,6 +164,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -188,14 +194,14 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
_is_async: bool = False,
api_version: str | None = None,
litellm_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
# Override to use Azure-specific client initialization
if isinstance(client, OpenAI) or isinstance(client, AsyncOpenAI):
client = None
return self.get_azure_openai_client(
litellm_params=litellm_params or {},
litellm_params=dict(litellm_params) if litellm_params is not None else None,
api_key=api_key,
api_base=api_base,
api_version=api_version,

39
litellm/llms/azure/responses/transformation.py
View file

@ -1,4 +1,7 @@
import re
from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
import httpx
@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -25,6 +29,34 @@ else:
_DECODED_JSON: Final = TypeAdapter(object)
_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
value: Final = litellm_params.get(key)
return value if isinstance(value, str) and value else None
def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None:
"""
Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment
pinned to one for chat gets the default Responses version unless it sets responses_api_version.
"""
api_version: Final = _str_param(litellm_params, "api_version")
if api_base is not None and "api-version" in httpx.URL(api_base).params:
return api_version
responses_api_version: Final = _str_param(litellm_params, "responses_api_version")
if responses_api_version is not None:
return responses_api_version
if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version):
return AZURE_DEFAULT_RESPONSES_API_VERSION
return api_version
def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Parameters not supported by Azure Responses API
AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- A complete URL string, e.g.,
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
return BaseAzureLLM.get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
api_base=resolved_api_base,
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)

11
litellm/llms/azure_ai/anthropic/count_tokens/handler.py
View file

@ -30,7 +30,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
self,
model: str,
messages: list[dict[str, JsonValue]],
api_key: str,
api_key: str | None,
api_base: str,
litellm_params: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
@ -80,6 +80,15 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
api_key=api_key,
litellm_params=litellm_params,
)
if not {"x-api-key", "Authorization"} & headers.keys():
raise AnthropicError(
status_code=401,
message=(
f"No Azure AI API key or Entra ID credential is set for model {model}, so tokens can't be "
"counted. Set api_key or an Entra ID credential on the deployment, or AZURE_AI_API_KEY or "
"AZURE_API_KEY in the proxy environment."
),
)
# Use LiteLLM's async httpx client
async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI)

36
litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py
View file

@ -7,17 +7,33 @@ from typing import Any, Final
from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
azure_ai_anthropic_count_tokens_handler: Final = AzureAIAnthropicCountTokensHandler()
def _missing_api_base_response(request_model: str, model_to_use: str) -> TokenCountResponse:
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="azure_ai_anthropic_api",
error=True,
error_message=(
f"No Azure AI API base is set for model {request_model or model_to_use}, so tokens can't be counted. "
"Set api_base on the deployment or AZURE_AI_API_BASE or AZURE_API_BASE in the proxy environment."
),
status_code=500,
)
class AzureAIAnthropicTokenCounter(BaseTokenCounter):
"""Token counter implementation for Azure AI Anthropic provider using the CountTokens API."""
@ -38,6 +54,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
system: object | None = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Azure AI Anthropic's CountTokens API.
@ -65,18 +82,15 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
if not api_key:
api_key = os.getenv("AZURE_AI_API_KEY")
# Get API base from deployment config or environment
api_base = litellm_params.get("api_base")
if not api_base:
api_base = os.getenv("AZURE_AI_API_BASE")
if not api_key:
verbose_logger.warning("No Azure AI API key found for token counting")
return None
api_base: Final = (
litellm_params.get("api_base")
or os.getenv("AZURE_AI_API_BASE")
or os.getenv("AZURE_API_BASE")
or litellm.api_base
)
if not api_base:
verbose_logger.warning("No Azure AI API base found for token counting")
return None
return _missing_api_base_response(request_model, model_to_use)
try:
result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(

9
litellm/llms/azure_ai/anthropic/count_tokens/transformation.py
View file

@ -24,7 +24,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
def get_required_headers(
self,
api_key: str,
api_key: str | None,
litellm_params: dict[str, object] | None = None,
) -> dict[str, str]:
"""
@ -45,12 +45,12 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
"Content-Type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
"x-api-key": api_key, # Azure AI Anthropic requires this header
**({"x-api-key": api_key} if api_key else {}),
}
# Also set up Azure auth headers for flexibility
litellm_params = litellm_params or {}
if "api_key" not in litellm_params:
if api_key and "api_key" not in litellm_params:
litellm_params["api_key"] = api_key
litellm_params_obj: Final = GenericLiteLLMParams.model_validate(litellm_params)
@ -58,8 +58,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
# Get Azure auth headers (api-key or Authorization)
azure_headers = BaseAzureLLM.base_validate_azure_environment(headers={}, litellm_params=litellm_params_obj)
# Merge Azure auth headers
headers.update(azure_headers)
if "api-key" in azure_headers:
headers.setdefault("x-api-key", azure_headers["api-key"])
return headers

29
litellm/llms/azure_ai/chat/transformation.py
View file

@ -1,6 +1,7 @@
import copy
import enum
import re
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Final, cast
import httpx
@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import (
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
from litellm.llms.openai.openai import OpenAIConfig
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
from litellm.llms.xai.chat.transformation import XAIChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse, ProviderField
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
from litellm.utils import add_path_to_api_base, supports_tool_choice
if TYPE_CHECKING:
@ -45,6 +46,14 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
)
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
if isinstance(usage, dict):
XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
return super().chunk_parser(chunk)
class AzureAIGPT5Config(OpenAIGPT5Config):
@classmethod
def _model_map_lookup_name(cls, model: str) -> str:
@ -319,7 +328,7 @@ class AzureAIStudioConfig(OpenAIConfig):
json_mode: bool | None = None,
) -> ModelResponse:
model_response.model = f"azure_ai/{model}"
return super().transform_response(
response: Final = super().transform_response(
model=model,
raw_response=raw_response,
model_response=model_response,
@ -332,6 +341,20 @@ class AzureAIStudioConfig(OpenAIConfig):
api_key=api_key,
json_mode=json_mode,
)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
return response
def get_model_response_iterator(
self,
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
sync_stream: bool,
json_mode: bool | None = False,
) -> "AzureAIChatCompletionResponseIterator":
return AzureAIChatCompletionResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
def should_retry_llm_api_inside_llm_translation_on_http_error(
self, e: httpx.HTTPStatusError, litellm_params: dict

26
litellm/llms/base_llm/base_model_iterator.py
View file

@ -6,16 +6,38 @@ import litellm
if TYPE_CHECKING:
import httpx
from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
from litellm.types.utils import (
Choices,
Delta,
GenericStreamingChunk,
Message,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
def _indexed_thinking_block(
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, index: int
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock:
if block["type"] == "redacted_thinking":
return {**block, "index": index}
return {**block, "index": index}
def _streaming_delta(message: Message) -> Delta:
thinking_blocks: Final = message.thinking_blocks if hasattr(message, "thinking_blocks") else None
return Delta(
**message.model_dump(exclude={"thinking_blocks"}),
thinking_blocks=(
None
if thinking_blocks is None
else [_indexed_thinking_block(block, index) for index, block in enumerate(thinking_blocks)]
),
)
def convert_model_response_to_streaming(
model_response: ModelResponse,
) -> ModelResponseStream:
@ -40,9 +62,7 @@ def convert_model_response_to_streaming(
streaming_choices.append(
StreamingChoices(
index=choice.index,
delta=Delta(
**cast(Choices, choice).message.model_dump(),
),
delta=_streaming_delta(cast(Choices, choice).message),
finish_reason=choice.finish_reason,
)
)

9
litellm/llms/base_llm/base_utils.py
View file

@ -16,7 +16,13 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import (
CountContextManagement,
Message,
ProviderSpecificModelInfo,
TokenCountMessagesFormat,
TokenCountResponse,
)
class BaseTokenCounter(ABC):
@ -32,6 +38,7 @@ class BaseTokenCounter(ABC):
system: Any | None = None,
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
pass

20
litellm/llms/base_llm/chat/transformation.py
View file

@ -8,7 +8,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from litellm.constants import DEFAULT_MAX_TOKENS, RESPONSE_FORMAT_TOOL_NAME
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -38,6 +38,9 @@ else:
LiteLLMLoggingObj = Any
REQUEST_PARAMS: Final = TypeAdapter(Mapping[str, object])
class BaseLLMException(Exception):
def __init__(
self,
@ -126,6 +129,17 @@ class BaseConfig(ABC):
thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None
def is_thinking_requested(self, non_default_params: Mapping[str, object]) -> bool:
match non_default_params:
case {"thinking": True | {"type": "enabled" | "adaptive"}}:
return True
case {"reasoning_effort": None | "none"}:
return False
case {"reasoning_effort": object()}:
return True
case _:
return False
def is_max_tokens_in_request(self, non_default_params: dict) -> bool:
"""
OpenAI spec allows max_tokens or max_completion_tokens to be specified.
@ -488,6 +502,10 @@ class BaseConfig(ABC):
"""
return True
@property
def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
return False
def post_stream_processing(self, stream: object) -> object:
"""Hook for providers to post-process streaming responses. Default: pass-through."""
return stream

1
litellm/llms/base_llm/containers/transformation.py
View file

@ -81,6 +81,7 @@ class BaseContainerConfig(ABC):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return {}

23
litellm/llms/base_llm/decisions/transformation.py
View file

@ -390,8 +390,24 @@ def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRe
)
_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...])
def _normalize_answer(value: object) -> Mapping[str, object]:
answer: Final = _MAPPING.validate_python(value)
legend: Final = answer.get("legend")
if answer.get("type") != "score" or not isinstance(legend, list):
return answer
labels: Final = _LABELS.validate_python(legend)
return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}}
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request)
response: Final = _MAPPING.validate_python(payload)
answers: Final = _MAPPING.validate_python(response.get("answers"))
normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}}
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request)
def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
@ -495,6 +511,11 @@ class BaseDecisionsConfig(ABC):
def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
return self.resolve_api_base(api_base), self.resolve_api_key(api_key)
def resolve_credential(
self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
) -> str | None:
return api_key
@staticmethod
def _first_secret(names: tuple[str, ...]) -> str | None:
return next((value for value in (get_secret_str(name) for name in names) if value), None)

5
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -203,6 +203,11 @@ class BaseTranslation(ABC):
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
return None
def needs_ended_stream_scan(self, guardrail: "CustomGuardrail", responses_so_far: Sequence[object]) -> bool:
"""Whether the end-of-stream round scans ``guardrail`` even when a sampled round already covered every
text, because this translation hands that guardrail marks only the ended round carries."""
return False
def released_stream_as_ended(self, responses_so_far: Sequence[object]) -> tuple[object, ...]:
"""The chunks a client left the stream with, closed the way this endpoint ends a stream, so the
end-of-stream scan also inspects tool calls the stream never finished"""

13
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -4,7 +4,7 @@ import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None:
return flat_name if isinstance(flat_name, str) else None
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
def list_field(data: object, key: str) -> Sequence[object]:
try:
return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key))
except ValidationError:
return ()
def anthropic_tool_names(tool: object) -> tuple[str, ...]:
"""Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks

10
litellm/llms/base_llm/realtime/transcription_protocol.py
View file

@ -102,6 +102,16 @@ def json_integer(
return value
def uses_server_vad(
update: TranscriptionSessionUpdate, backend: str, drop_params: bool, error: ProtocolErrorType
) -> bool:
if update.turn_detection_disabled:
return False
if update.turn_detection_type in (None, "server_vad") or drop_params:
return True
raise error(f"{backend} supports server_vad turn detection or null. Set drop_params to use server_vad instead")
def new_event_id() -> str:
return f"event_{uuid.uuid4().hex}"

123
litellm/llms/base_llm/videos/async_task_utils.py Normal file
View file

@ -0,0 +1,123 @@
import base64
from collections.abc import Mapping
from io import BufferedReader, BytesIO
from itertools import accumulate, repeat
from pathlib import Path, PurePath
from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.images.utils import ImageEditRequestUtils
from litellm.types.videos.utils import decode_video_id_with_provider
_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
def _fields(value: object) -> Mapping[str, object] | None:
try:
return _FIELDS_ADAPTER.validate_python(value)
except ValidationError:
return None
def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
return _fields(getattr(logging_obj, "litellm_params", None)) or {}
def _id_safe_name(name: object) -> str | None:
return name if isinstance(name, str) and name and ";" not in name else None
def _stamped_models(metadata: object) -> tuple[str, str | None] | None:
fields: Final = _fields(metadata)
if fields is None:
return None
requested: Final = _id_safe_name(fields.get("model_group"))
served: Final = _id_safe_name(fields.get("deployment_model_name"))
if requested is None:
return None
return requested, None if served is None or served == requested or "*" in served else served
def routed_video_models(logging_obj: object) -> tuple[str, str | None] | None:
"""
A later status or content request checks the key's model access against the model encoded in the id, and
routes by it. One upstream model name can be served by several deployments (regions, accounts), so only the
group that served the task can send its polls back to the deployment that created it, while a key may be
scoped to the alias, routing group or wildcard name it asked for instead. The id carries both names: the one
the router was asked for on this hop, and the group that served it when that differs and is not a wildcard.
The router stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
"""
params: Final = _logged_litellm_params(logging_obj)
stamped: Final = (_stamped_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
return next((models for models in stamped if models), None)
def polled_video_models(logging_obj: object) -> tuple[str | None, str | None]:
"""
The served group survives into the returned id only when the router sent this poll to it, so an edited id
cannot write any name it likes back.
"""
video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
if not isinstance(video_id, str):
return None, None
decoded: Final = decode_video_id_with_provider(video_id)
served: Final = decoded.get("served_model")
routed: Final = routed_video_models(logging_obj)
return decoded.get("model_id") or None, served if routed is None or served in routed else None
def _tuple_payload(file_obj: object) -> object:
if not isinstance(file_obj, tuple):
return file_obj
file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
return file_tuple[1] if len(file_tuple) >= 2 else None
def _file_content(file_obj: object) -> object:
unwrapped: Final = accumulate(repeat(None), lambda current, _: _tuple_payload(current), initial=file_obj)
return next(item for item in unwrapped if not isinstance(item, tuple))
def input_reference_bytes(file_obj: object) -> bytes:
content: Final = _file_content(file_obj)
if isinstance(content, PurePath):
return Path(content).read_bytes()
if isinstance(content, (BytesIO, BufferedReader)):
current_position: Final = content.tell()
_ = content.seek(0)
data: Final = content.read()
_ = content.seek(current_position)
return data
if isinstance(content, bytes):
return content
if isinstance(content, bytearray):
return bytes(content)
read: Final = getattr(content, "read", None)
if callable(read):
read_data: Final[object] = read()
if isinstance(read_data, bytes):
return read_data
raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
def _reference_object_url(reference: Mapping[str, object]) -> str:
image_url: Final = reference.get("image_url")
if isinstance(image_url, str) and image_url:
return image_url
raise ValueError("input_reference as an object must carry an image_url, a file_id is not supported")
def input_reference_url(image: object) -> str:
"""A string is a URL the provider fetches itself and is never opened as a local path. OpenAI's object form
carries that URL, or a data URL, in ``image_url``."""
if isinstance(image, str):
return image
reference: Final = _fields(image)
if reference is not None:
return _reference_object_url(reference)
content: Final = input_reference_bytes(image)
content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
encoded: Final = base64.b64encode(content).decode("utf-8")
return f"data:{content_type};base64,{encoded}"

28
litellm/llms/bedrock/base_aws_llm.py
View file

@ -99,11 +99,16 @@ def _assume_role_params(
)
_SecureTransportBool = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
_SecureTransportBool: Final = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
_CalledViaLastEquals: Final = TypedDict(
"_CalledViaLastEquals", {"aws:CalledViaLast": ReadOnly[Literal["aws-external-anthropic.amazonaws.com"]]}
)
class _SecureTransportCondition(TypedDict):
class _SessionPolicyCondition(TypedDict):
Bool: ReadOnly[_SecureTransportBool]
StringEquals: NotRequired[ReadOnly[_CalledViaLastEquals]]
class _SessionPolicyStatement(TypedDict):
@ -111,7 +116,7 @@ class _SessionPolicyStatement(TypedDict):
Effect: ReadOnly[Literal["Allow"]]
Action: ReadOnly[tuple[str, ...]]
Resource: ReadOnly[Literal["*"]]
Condition: ReadOnly[_SecureTransportCondition]
Condition: ReadOnly[_SessionPolicyCondition]
class WebIdentitySessionPolicy(TypedDict):
@ -147,11 +152,24 @@ _WEB_IDENTITY_SESSION_POLICY_ACTIONS: Final[Mapping[str, tuple[str, ...]]] = Map
"aws-external-anthropic:Get*",
"aws-external-anthropic:List*",
),
"ClaudePlatformOutboundIdentityLiteLLM": (
"sts:GetWebIdentityToken",
"sts:TagGetWebIdentityToken",
),
"BedrockMantleLiteLLM": ("bedrock-mantle:CreateInference",),
}
)
_SECURE_TRANSPORT_ONLY: Final = _SecureTransportCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
_SECURE_TRANSPORT_ONLY: Final = _SessionPolicyCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
_WEB_IDENTITY_SESSION_POLICY_CONDITIONS: Final[Mapping[str, _SessionPolicyCondition]] = MappingProxyType(
{
"ClaudePlatformOutboundIdentityLiteLLM": _SessionPolicyCondition(
Bool=_SecureTransportBool({"aws:SecureTransport": "true"}),
StringEquals=_CalledViaLastEquals({"aws:CalledViaLast": "aws-external-anthropic.amazonaws.com"}),
),
}
)
def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
@ -163,7 +181,7 @@ def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
Effect="Allow",
Action=actions,
Resource="*",
Condition=_SECURE_TRANSPORT_ONLY,
Condition=_WEB_IDENTITY_SESSION_POLICY_CONDITIONS.get(sid, _SECURE_TRANSPORT_ONLY),
)
for sid, actions in _WEB_IDENTITY_SESSION_POLICY_ACTIONS.items()
),

19
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -1,5 +1,5 @@
import json
from collections.abc import Iterator
from collections.abc import Iterator, Mapping
from typing import Final
import httpx
@ -10,6 +10,7 @@ from litellm.anthropic_beta_headers_manager import (
)
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.base_llm import base_model_iterator
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -21,7 +22,7 @@ from litellm.utils import CustomStreamWrapper
from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token, pop_aws_auth_params, run_aws_signing
from ..common_utils import BedrockError, error_response_text, get_all_bedrock_regions
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
from .invoke_handler import AWSEventStreamDecoder, make_call
def make_sync_call(
@ -36,7 +37,8 @@ def make_sync_call(
fake_stream: bool = False,
stream_chunk_size: int | None = None,
timeout: float | httpx.Timeout | None = None,
) -> tuple[MockResponseIterator | Iterator[object], httpx.Headers]:
optional_params: Mapping[str, object] | None = None,
) -> tuple[base_model_iterator.MockResponseIterator | Iterator[object], httpx.Headers]:
if client is None:
client = get_httpx_client() # Create a new client if none provided
@ -64,14 +66,15 @@ def make_sync_call(
model_response=litellm.ModelResponse(),
stream=True,
logging_obj=logging_obj,
optional_params={},
optional_params=dict(optional_params or {}),
api_key="",
data=data,
messages=messages,
encoding=litellm.encoding,
json_mode=json_mode,
)
completion_stream: MockResponseIterator | Iterator[object] = MockResponseIterator(
model_response=model_response, json_mode=json_mode
completion_stream: base_model_iterator.MockResponseIterator | Iterator[object] = (
base_model_iterator.MockResponseIterator(model_response=model_response)
)
else:
decoder: Final = AWSEventStreamDecoder(model=model, json_mode=json_mode)
@ -158,6 +161,7 @@ class BedrockConverseLLM(BaseAWSLLM):
json_mode=json_mode,
stream_chunk_size=stream_chunk_size,
timeout=timeout,
optional_params=optional_params,
)
streaming_response: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -290,7 +294,7 @@ class BedrockConverseLLM(BaseAWSLLM):
stream_chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size if stream is True else None
unencoded_model_id: Final = optional_params.pop("model_id", None)
fake_stream = optional_params.pop("fake_stream", False)
json_mode: Final = optional_params.get("json_mode", False)
json_mode: Final = optional_params.get("json_mode") is True
if unencoded_model_id is not None:
modelId = self.encode_model_id(model_id=unencoded_model_id)
else:
@ -463,6 +467,7 @@ class BedrockConverseLLM(BaseAWSLLM):
fake_stream=fake_stream,
stream_chunk_size=stream_chunk_size,
timeout=timeout,
optional_params=optional_params,
)
streaming_response: Final = CustomStreamWrapper(
completion_stream=completion_stream,

111
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -29,6 +29,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_lookaround_regex_patterns,
is_object_schema,
native_redacted_thinking_data,
parse_content_for_reasoning,
tool_with_sanitized_parameters,
@ -53,9 +54,16 @@ from litellm.llms.anthropic.chat.transformation import (
tools_without_unsupported_tool_search,
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround, bedrock_supports_tool_search
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_supports_tool_search,
deployment_model_info,
)
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
@ -939,17 +947,22 @@ class AmazonConverseConfig(BaseConfig):
return _tool
@staticmethod
def _supports_native_structured_outputs(model: str, custom_llm_provider: str | None = None) -> bool:
"""Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).
def _supports_native_structured_outputs(
model: str,
custom_llm_provider: str | None,
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> bool:
"""Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
Delegates to the standard ``supports_native_structured_output`` utility
which looks up the flag in ``litellm.model_cost`` via
``_get_model_info_helper``.
``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the
request asks for thinking, which the json tool path would drop.
Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
"""
from litellm.utils import supports_native_structured_output
return supports_native_structured_output(model=model, custom_llm_provider=custom_llm_provider)
return bedrock_chat_uses_native_structured_output(
model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info
)
@staticmethod
def _add_additional_properties_to_schema(schema: dict) -> dict:
@ -964,9 +977,9 @@ class AmazonConverseConfig(BaseConfig):
if not isinstance(schema, dict):
return schema
result: Final = dict(schema)
result: Final[dict[str, object]] = dict(schema)
if result.get("type") == "object" and "additionalProperties" not in result:
if is_object_schema(result) and "additionalProperties" not in result:
result["additionalProperties"] = False
# Recurse into nested schemas
@ -977,16 +990,46 @@ class AmazonConverseConfig(BaseConfig):
if "items" in result and isinstance(result["items"], dict):
result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"])
for defs_key in ("$defs", "definitions"):
if defs_key in result and isinstance(result[defs_key], dict):
if isinstance(definitions := result.get(defs_key), dict):
result[defs_key] = {
k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items()
k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in definitions.items()
}
for key in ("anyOf", "allOf", "oneOf"):
if key in result and isinstance(result[key], list):
result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]]
if isinstance(variants := result.get(key), list):
result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in variants]
return result
def _native_output_schema(
self,
model: str,
json_schema: Mapping[str, object],
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
if not self._supports_native_structured_outputs(
model, self.custom_llm_provider, thinking_requested, model_info
):
return None
if BedrockModelInfo.get_base_model(model).startswith("anthropic"):
return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse")
return json_schema
def _output_config_for_anthropic_format(
self, model: str, output_config_format: Mapping[str, object], model_info: Mapping[str, object] | None
) -> OutputConfigBlock | None:
native_format: Final = bedrock_native_output_format(
model, output_config_format, self.custom_llm_provider or "bedrock_converse", model_info
)
if native_format is None:
return None
schema, name, description = (native_format.get(key) for key in ("schema", "name", "description"))
return AmazonConverseConfig._create_output_config_for_response_format(
json_schema={**schema} if isinstance(schema, Mapping) else None,
name=name if isinstance(name, str) else None,
description=description if isinstance(description, str) else None,
)
@staticmethod
def _create_output_config_for_response_format(
json_schema: dict | None = None,
@ -1046,6 +1089,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
model: str,
drop_params: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
base_model: Final = BedrockModelInfo.get_base_model(model)
@ -1062,6 +1106,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params=optional_params,
non_default_params=non_default_params,
is_thinking_enabled=is_thinking_enabled,
model_info=model_info,
)
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["maxTokens"] = (
@ -1273,6 +1318,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
non_default_params: dict,
is_thinking_enabled: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
"""
Handles translation of response_format parameter to Bedrock format.
@ -1296,13 +1342,23 @@ class AmazonConverseConfig(BaseConfig):
if "type" in value and value["type"] == "text":
return optional_params
if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None:
native_schema: Final = (
self._native_output_schema(
model,
json_schema,
self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)),
model_info,
)
if json_schema is not None
else None
)
if native_schema is not None:
# Use Bedrock's native structured outputs API (outputConfig.textFormat)
# No synthetic tool injection, no fake_stream needed.
# Requires an explicit schema — json_object with no schema falls through
# to the tool-call path below.
output_config: Final = self._create_output_config_for_response_format(
json_schema=json_schema,
json_schema=dict(native_schema),
name=name,
description=description,
)
@ -1644,7 +1700,11 @@ class AmazonConverseConfig(BaseConfig):
tool_choice.pop("type", None)
def _prepare_request_params(
self, optional_params: dict, model: str, drop_params: bool = False
self,
optional_params: dict,
model: str,
drop_params: bool = False,
model_info: Mapping[str, object] | None = None,
) -> tuple[dict, dict, dict, OutputConfigBlock | None]:
"""Prepare and separate request parameters."""
# Consume the internal ``_output_config_normalized`` marker set by
@ -1687,17 +1747,12 @@ class AmazonConverseConfig(BaseConfig):
and output_config_format is not None
and output_config_format.get("type") == "json_schema"
and base_model.startswith("anthropic")
and self._supports_native_structured_outputs(model, self.custom_llm_provider)
):
output_config = self._create_output_config_for_response_format(
json_schema=output_config_format.get("schema"),
name=output_config_format.get("name"),
description=output_config_format.get("description"),
)
elif output_config is None and output_config_format is not None:
output_config = self._output_config_for_anthropic_format(model, output_config_format, model_info)
if output_config is None and output_config_format is not None:
litellm.verbose_logger.warning(
"Bedrock Converse: dropping `output_config.format` for model=%s — "
"model does not advertise `supports_native_structured_output` in "
"model does not take native structured output for this schema per "
"model_prices_and_context_window.json. The schema will not be "
"enforced; pass `response_format` to use the synthetic tool-call "
"fallback.",
@ -1987,7 +2042,7 @@ class AmazonConverseConfig(BaseConfig):
additional_request_params,
request_metadata,
output_config,
) = self._prepare_request_params(optional_params, model, drop_params)
) = self._prepare_request_params(optional_params, model, drop_params, deployment_model_info(litellm_params))
original_tools: Final = inference_params.pop("tools", [])

153
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -1,7 +1,7 @@
import json
import types
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Final, cast
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import TypeAdapter, ValidationError
@ -13,12 +13,14 @@ from litellm.caching.caching import InMemoryCache
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data
from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
from litellm.llms.anthropic.chat.transformation import JsonModeRequest
from litellm.llms.base_llm import base_model_iterator
from litellm.llms.base_llm.base_utils import ResponseFormatSchema
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -31,12 +33,8 @@ from litellm.types.llms.openai import (
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionUsageBlock,
)
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
Delta,
ModelResponse,
ModelResponseStream,
@ -227,7 +225,10 @@ async def make_call(
bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
stream_chunk_size: int | None = None,
timeout: float | httpx.Timeout | None = None,
) -> "tuple[MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
optional_params: Mapping[str, object] | None = None,
) -> tuple[
base_model_iterator.MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict], httpx.Headers
]:
try:
if client is None:
client = get_async_httpx_client(
@ -263,15 +264,16 @@ async def make_call(
model_response=litellm.ModelResponse(),
stream=True,
logging_obj=logging_obj,
optional_params={},
optional_params=dict(optional_params or {}),
api_key="",
data=data,
messages=messages,
encoding=litellm.encoding,
json_mode=json_mode,
)
completion_stream: MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict] = (
MockResponseIterator(model_response=model_response, json_mode=json_mode)
)
completion_stream: (
base_model_iterator.MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict]
) = base_model_iterator.MockResponseIterator(model_response=model_response)
else:
decoder: Final = _stream_decoder(
bedrock_invoke_provider,
@ -323,7 +325,7 @@ def make_sync_call(
bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
stream_chunk_size: int | None = None,
timeout: float | httpx.Timeout | None = None,
) -> "tuple[MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
) -> tuple[base_model_iterator.MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]:
try:
if client is None:
client = get_httpx_client(
@ -363,10 +365,11 @@ def make_sync_call(
data=data,
messages=messages,
encoding=litellm.encoding,
json_mode=json_mode,
)
completion_stream: MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict] = (
MockResponseIterator(model_response=model_response, json_mode=json_mode)
)
completion_stream: (
base_model_iterator.MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict]
) = base_model_iterator.MockResponseIterator(model_response=model_response)
else:
decoder: Final = _stream_decoder(
bedrock_invoke_provider,
@ -448,6 +451,7 @@ class _EventStreamTally:
f"x-amzn-requestid={_response_header(response_headers, 'x-amzn-requestid')!r}, "
f"{self.bytes_received} bytes received"
)
request_id_headers: Final = provider_request_id_headers(response_headers)
if not self.events:
return BedrockError(
status_code=502,
@ -455,6 +459,7 @@ class _EventStreamTally:
"Bedrock answered the stream with HTTP 200 but its body decoded to no events "
f"({detail}, first bytes={self.head!r})"
),
headers=request_id_headers,
)
if not self.recognized_events:
return BedrockError(
@ -464,10 +469,12 @@ class _EventStreamTally:
f"event type (event types={sorted(self.unrecognized_event_types)}, {detail}, "
f"first payload={self.unrecognized_head!r})"
),
headers=request_id_headers,
)
return BedrockError(
status_code=502,
message=f"Bedrock stream ended with {undecoded} undecoded bytes after {self.events} events ({detail})",
headers=request_id_headers,
)
@ -837,11 +844,12 @@ class AWSEventStreamDecoder:
event_stream_buffer: Final = EventStreamBuffer()
tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses())
request_id_headers: Final = provider_request_id_headers(response_headers)
for chunk in iterator:
event_stream_buffer.add_data(chunk)
tally.add_chunk(chunk)
for event in event_stream_buffer:
message = self._decode_event(event, tally)
message = self._decode_event(event, tally, request_id_headers)
if message:
# sse_event = ServerSentEvent(data=message, event="completion")
_data = json.loads(message)
@ -858,11 +866,12 @@ class AWSEventStreamDecoder:
event_stream_buffer: Final = EventStreamBuffer()
tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses())
request_id_headers: Final = provider_request_id_headers(response_headers)
async for chunk in iterator:
event_stream_buffer.add_data(chunk)
tally.add_chunk(chunk)
for event in event_stream_buffer:
message = self._decode_event(event, tally)
message = self._decode_event(event, tally, request_id_headers)
if message:
_data = json.loads(message)
yield self.chunk_parser(chunk_data=_data)
@ -882,27 +891,34 @@ class AWSEventStreamDecoder:
)
return response_stream_shape
def _decode_event(self, event: "EventStreamMessage", tally: _EventStreamTally) -> str | None:
def _decode_event(
self, event: "EventStreamMessage", tally: _EventStreamTally, request_id_headers: httpx.Headers | None
) -> str | None:
response_stream_shape: Final = self._response_stream_shape()
response_dict: Final = bedrock_event_stream_response(event)
tally.add_event(event, response_dict["headers"])
return self._parse_message_from_response(response_dict, response_stream_shape)
return self._parse_message_from_response(response_dict, response_stream_shape, request_id_headers)
def _parse_message_from_event(self, event: "EventStreamMessage") -> str | None:
response_stream_shape: Final = self._response_stream_shape()
return self._parse_message_from_response(bedrock_event_stream_response(event), response_stream_shape)
def _parse_message_from_response(
self, response_dict: BedrockEventStreamResponseDict, response_stream_shape: "Shape"
self,
response_dict: BedrockEventStreamResponseDict,
response_stream_shape: "Shape",
request_id_headers: httpx.Headers | None = None,
) -> str | None:
parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
raise build_bedrock_stream_error(response_dict, response_stream_shape)
raise build_bedrock_stream_error(response_dict, response_stream_shape, headers=request_id_headers)
event_type: Final = bedrock_event_stream_header(response_dict["headers"], ":event-type")
event_error_status: Final = bedrock_stream_event_error_status(event_type)
if event_type is not None and event_error_status is not None:
raise build_bedrock_stream_event_error(event_type, event_error_status, response_dict["body"])
raise build_bedrock_stream_event_error(
event_type, event_error_status, response_dict["body"], headers=request_id_headers
)
if "chunk" in parsed_response:
chunk = parsed_response.get("chunk")
if not chunk:
@ -977,98 +993,3 @@ class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)
class MockResponseIterator: # for returning ai21 streaming responses
def __init__(self, model_response, json_mode: bool | None = False):
self.model_response = model_response
self.json_mode = json_mode
self.is_done = False
# Sync iterator
def __iter__(self):
return self
def _handle_json_mode_chunk(
self, text: str, tool_calls: list[ChatCompletionToolCallChunk] | None
) -> tuple[str, ChatCompletionToolCallChunk | None]:
"""
If JSON mode is enabled, convert the tool call to a message.
Bedrock returns the JSON schema as part of the tool call
OpenAI returns the JSON schema as part of the content, this handles placing it in the content
Args:
text: str
tool_use: Optional[ChatCompletionToolCallChunk]
Returns:
Tuple[str, Optional[ChatCompletionToolCallChunk]]
text: The text to use in the content
tool_use: The ChatCompletionToolCallChunk to use in the chunk response
"""
tool_use: ChatCompletionToolCallChunk | None = None
if self.json_mode is True and tool_calls is not None:
message: Final = litellm.AnthropicConfig().convert_tool_response_to_message(tool_calls=tool_calls)
if message is not None:
text = message.content or ""
tool_use = None
elif tool_calls is not None and len(tool_calls) > 0:
tool_use = tool_calls[0]
return text, tool_use
def _chunk_parser(self, chunk_data: ModelResponse) -> GChunk:
try:
chunk_usage: Final[Usage] = getattr(chunk_data, "usage")
text = chunk_data.choices[0].message.content or ""
tool_use = None
_model_response_tool_call: Final = cast(
list[ChatCompletionMessageToolCall] | None,
cast(Choices, chunk_data.choices[0]).message.tool_calls,
)
if self.json_mode is True:
text, tool_use = self._handle_json_mode_chunk(
text=text,
tool_calls=chunk_data.choices[0].message.tool_calls,
)
elif _model_response_tool_call is not None:
tool_use = ChatCompletionToolCallChunk(
id=_model_response_tool_call[0].id,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=_model_response_tool_call[0].function.name,
arguments=_model_response_tool_call[0].function.arguments,
),
index=0,
)
processed_chunk: Final = GChunk(
text=text,
tool_use=tool_use,
is_finished=True,
finish_reason=map_finish_reason(finish_reason=chunk_data.choices[0].finish_reason or ""),
usage=ChatCompletionUsageBlock(
prompt_tokens=chunk_usage.prompt_tokens,
completion_tokens=chunk_usage.completion_tokens,
total_tokens=chunk_usage.total_tokens,
),
index=0,
)
return processed_chunk
except Exception as e:
raise ValueError(f"Failed to decode chunk: {chunk_data}. Error: {e}")
def __next__(self):
if self.is_done:
raise StopIteration
self.is_done = True
return self._chunk_parser(self.model_response)
# Async iterator
def __aiter__(self):
return self
async def __anext__(self):
if self.is_done:
raise StopAsyncIteration
self.is_done = True
return self._chunk_parser(self.model_response)

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -92,8 +92,11 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
optional_params: dict,
model: str,
drop_params: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
return AmazonConverseConfig.map_openai_params(self, non_default_params, optional_params, model, drop_params)
return AmazonConverseConfig.map_openai_params(
self, non_default_params, optional_params, model, drop_params, model_info
)
def transform_request(
self,

31
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -23,7 +23,10 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
)
from litellm.llms.bedrock.common_utils import (
apply_bedrock_invoke_structured_output,
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_supports_tool_search,
deployment_model_info,
get_anthropic_beta_from_headers,
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
@ -32,6 +35,7 @@ from litellm.llms.bedrock.common_utils import (
tools_without_eager_input_streaming,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
AnthropicThinkingParam,
@ -48,6 +52,7 @@ else:
LiteLLMLoggingObj = Any
_NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
@ -70,6 +75,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
anthropic_version: str = "bedrock-2023-05-31"
def __init__(self, model_info: Mapping[str, object] | None = None) -> None:
super().__init__()
self._deployment_model_info: Final = model_info
@property
def custom_llm_provider(self) -> str | None:
return "bedrock"
@ -80,10 +89,18 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
def get_supported_openai_params(self, model: str) -> list[str]:
return AnthropicConfig.get_supported_openai_params(self, model)
def uses_native_structured_output(self, model: str) -> bool:
from litellm.utils import supports_native_structured_output
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return AnthropicConfig.native_output_schema(json_schema)
return supports_native_structured_output(model=model, custom_llm_provider="bedrock")
def uses_native_structured_output_for_schema(
self,
model: str,
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
return bedrock_chat_uses_native_structured_output(
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
def map_openai_params(
self,
@ -239,6 +256,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
apply_bedrock_invoke_structured_output(
model=model,
request_body=anthropic_request,
model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)),
)
strip_unsupported_bedrock_invoke_output_config_keys(
model=model,
@ -270,7 +288,12 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking)
user_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
client_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
user_beta_set: Final = (
client_beta_set
if "safeguards" in optional_params
else client_beta_set - {ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value}
)
beta_set: Final = set(user_beta_set)
auto_betas: Final = self.get_anthropic_beta_list(
model=model,

11
litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py
View file

@ -14,9 +14,9 @@ from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.factory import (
cohere_message_pt,
custom_prompt,
deepseek_r1_pt,
prompt_factory,
render_custom_prompt_template,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
@ -601,7 +601,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
return None
def convert_messages_to_prompt(self, model, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
# handle anthropic prompts and amazon titan prompts
prompt = ""
chat_history: list | None = None
@ -609,11 +609,8 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
if model in custom_prompt_dict:
# check if the model has a registered custom prompt
model_prompt_details: Final = custom_prompt_dict[model]
prompt = custom_prompt(
role_dict=model_prompt_details["roles"],
initial_prompt_value=model_prompt_details.get("initial_prompt_value", ""),
final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
messages=messages,
prompt = render_custom_prompt_template(
template=model_prompt_details, messages=messages, model=model, llm_provider="bedrock"
)
return prompt, None
## ELSE

119
litellm/llms/bedrock/common_utils.py
View file

@ -29,6 +29,7 @@ import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -249,15 +250,99 @@ def convert_bedrock_invoke_output_format_to_inline_schema(
request_body["messages"] = new_messages
def _bedrock_model_supports(model: str, key: str) -> bool:
def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "bedrock") -> bool:
from litellm.utils import supports_factory
return supports_factory(model=model, custom_llm_provider="bedrock", key=key)
return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key)
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
def deployment_model_info(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]:
model_info: Final = (litellm_params or {}).get("model_info")
return _DEPLOYMENT_MODEL_INFO.validate_python(model_info) if isinstance(model_info, Mapping) else {}
def _bedrock_native_output_opt_in(
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None
) -> bool:
deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output")
if isinstance(deployment_flag, bool):
return deployment_flag
return _bedrock_model_supports(model, "supports_bedrock_native_structured_output", custom_llm_provider)
def bedrock_supports_native_structured_output(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return _bedrock_model_supports(
model, "supports_native_structured_output", custom_llm_provider
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
def bedrock_chat_uses_native_structured_output(
model: str,
thinking_requested: bool,
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> bool:
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or (
thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
)
BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64
def bedrock_cleaned_output_schema(
model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock"
) -> Mapping[str, object] | None:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if (
not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
and schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
):
return None
return AnthropicConfig.native_output_schema(schema)
def bedrock_native_output_schema(
model: str,
schema: Mapping[str, object],
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> Mapping[str, object] | None:
native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider)
if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider):
return schema if native_schema is None else native_schema
if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info):
return native_schema
return None
def bedrock_native_output_format(
model: str,
output_format: Mapping[str, object],
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> Mapping[str, object] | None:
schema: Final = output_format.get("schema")
if not isinstance(schema, Mapping):
return (
output_format if bedrock_supports_native_structured_output(model, custom_llm_provider, model_info) else None
)
native_schema: Final = bedrock_native_output_schema(model, schema, custom_llm_provider, model_info)
if native_schema is None:
return None
return {**output_format, "schema": native_schema}
def apply_bedrock_invoke_structured_output(
model: str,
request_body: dict[str, object], # mutable-ok: edited in place like siblings
model_info: Mapping[str, object] | None = None,
) -> None:
"""
Route Anthropic structured-output params to what the Bedrock model supports.
@ -277,17 +362,18 @@ def apply_bedrock_invoke_structured_output(
if schema_format is None:
return
if _bedrock_model_supports(model, "supports_native_structured_output"):
native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info)
if native_format is not None:
existing_output_config: Final = request_body.get("output_config")
if isinstance(existing_output_config, dict):
existing_output_config["format"] = schema_format
existing_output_config["format"] = native_format
else:
request_body["output_config"] = {"format": schema_format} # rebind-ok: out-param
request_body["output_config"] = {"format": native_format} # rebind-ok: out-param
return
verbose_logger.warning(
"Bedrock Invoke: model=%s does not advertise `supports_native_structured_output` "
"in model_prices_and_context_window.json, so the JSON schema was inlined into "
"Bedrock Invoke: model=%s does not take native structured output for this schema "
"per model_prices_and_context_window.json, so the JSON schema was inlined into "
"the last user message and is NOT enforced by the model.",
model,
)
@ -1296,7 +1382,6 @@ def is_claude_4_5_on_bedrock(model: str) -> bool:
_BEDROCK_MODEL_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
def bedrock_converse_supports_strict_tools(model: str) -> bool:
@ -1327,8 +1412,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
"""
params: Final = litellm_params or {}
model_info: Final = _DEPLOYMENT_MODEL_INFO.validate_python(params.get("model_info") or {})
deployment_flag: Final = model_info.get("supports_regex_lookaround")
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
if isinstance(deployment_flag, bool):
return deployment_flag
base_model: Final = params.get("base_model")
@ -1913,13 +1997,18 @@ def bedrock_event_stream_header(headers: Mapping[str, object], name: str) -> str
return value if isinstance(value, str) else None
def build_bedrock_stream_event_error(event_type: str, status_code: int, body: bytes) -> BedrockError:
return BedrockError(status_code=status_code, message=f"{event_type} {body.decode(errors='replace')}")
def build_bedrock_stream_event_error(
event_type: str, status_code: int, body: bytes, headers: httpx.Headers | None = None
) -> BedrockError:
return BedrockError(
status_code=status_code, message=f"{event_type} {body.decode(errors='replace')}", headers=headers
)
def build_bedrock_stream_error(
response_dict: BedrockEventStreamResponseDict,
response_stream_shape: Shape | None,
headers: httpx.Headers | None = None,
) -> BedrockError:
"""Build a BedrockError for a non-200 event-stream error event.
@ -1929,12 +2018,14 @@ def build_bedrock_stream_error(
"""
exception_type: Final = bedrock_event_stream_header(response_dict["headers"], ":exception-type")
if exception_type is None:
return BedrockError(status_code=response_dict["status_code"], message=response_dict["body"].decode())
return BedrockError(
status_code=response_dict["status_code"], message=response_dict["body"].decode(), headers=headers
)
member: Final = _structure_members(response_stream_shape).get(exception_type)
modeled_status: Final = None if member is None else _modeled_error_status(member)
status_code: Final = response_dict["status_code"] if modeled_status is None else modeled_status
return build_bedrock_stream_event_error(exception_type, status_code, response_dict["body"])
return build_bedrock_stream_event_error(exception_type, status_code, response_dict["body"], headers=headers)
class BedrockEventStreamDecoderBase:

9
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
View file

@ -16,7 +16,13 @@ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import LiteLLMPydanticObjectBase, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import (
CountContextManagement,
LiteLLMPydanticObjectBase,
LlmProviders,
TokenCountMessagesFormat,
TokenCountResponse,
)
RUNTIME_TOKENIZER_TYPE: Final = "bedrock_api"
MANTLE_TOKENIZER_TYPE: Final = "bedrock_mantle_api"
@ -109,6 +115,7 @@ class BedrockTokenCounter(BaseTokenCounter):
system: object | None = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
if not messages:
return None

19
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -3,6 +3,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -69,6 +70,9 @@ else:
LiteLLMLoggingObj = Any
_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
class AmazonAnthropicClaudeMessagesConfig(
AnthropicMessagesConfig,
AmazonInvokeConfig,
@ -90,6 +94,9 @@ class AmazonAnthropicClaudeMessagesConfig(
def beta_headers_provider(self) -> str:
return self.custom_llm_provider or "bedrock"
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return True
BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
def get_error_class(
@ -552,8 +559,17 @@ class AmazonAnthropicClaudeMessagesConfig(
if anthropic_model_info.is_eager_input_streaming_used(tools):
beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
dangerous_tool_use_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
if anthropic_messages_optional_request_params.get("safeguards") is not None:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
beta_set.add(dangerous_tool_use_beta)
elif dangerous_tool_use_beta in beta_set and self.drops_dangerous_tool_use_beta_without_safeguards():
beta_set.discard(dangerous_tool_use_beta)
verbose_logger.debug(
"Bedrock Invoke: dropping %s from client headers because the request has no safeguards. Bedrock "
"answers it with 'invalid beta flag' next to structured outputs before Opus 4.7, and on Sonnet 4.5, "
"Opus 4.5, and Opus 4.6 even alone.",
dangerous_tool_use_beta,
)
self._filter_context_management_for_bedrock_invoke(
anthropic_messages_request=anthropic_messages_request,
@ -739,6 +755,7 @@ class AmazonAnthropicClaudeMessagesConfig(
apply_bedrock_invoke_structured_output(
model=model,
request_body=anthropic_messages_request,
model_info=_MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None)),
)
normalize_bedrock_opus_output_config_effort(
model=model,

3
litellm/llms/bedrock/messages/mantle_transformation.py
View file

@ -67,6 +67,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
def should_filter_anthropic_beta_headers(self) -> bool:
return False
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return False
def get_complete_url(
self,
api_base: str | None,

519
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -1,17 +1,23 @@
import copy
import json
from collections.abc import Mapping, Sequence
import struct
from binascii import crc32
from collections.abc import AsyncGenerator, AsyncIterable, Mapping, Sequence
from dataclasses import dataclass
from functools import partial, reduce
from itertools import accumulate, chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias
from pydantic import InstanceOf, TypeAdapter
from pydantic import InstanceOf, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
StreamTransformSink,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
@ -40,7 +46,7 @@ from litellm.proxy.guardrails.auto_router_compression import is_compression_guar
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.pass_through.guardrail_translation.handler import (
PassThroughEndpointHandler,
@ -584,8 +590,6 @@ class _StreamFrame(TypedDict):
def _unpack_uint32(buffer: bytes) -> int:
import struct
return struct.unpack("!I", buffer)[0]
@ -620,6 +624,307 @@ def _collect_stream_delta_text_holders(delta: object) -> list[_DeltaHolder]:
return holders
def _delta_frame_texts(payload: Mapping[str, object]) -> tuple[tuple[_StreamGroupKey, str], ...]:
"""The guardrailable texts of one ``contentBlockDelta`` payload, each keyed by its content block and kind."""
block_index: Final = payload.get("contentBlockIndex")
# Group per content block: a guardrail edit that changes length must not move text between blocks.
return tuple(
((block_index if isinstance(block_index, int) else None, group_key), container[key])
for group_key, container, key in _collect_stream_delta_text_holders(payload.get("delta"))
)
@dataclass(frozen=True, slots=True)
class _StreamTextGroups:
"""Stream texts joined per group, keeping where each fragment came from so a rewrite can be split back."""
groups: tuple[_StreamGroupKey, ...]
members: Mapping[_StreamGroupKey, Sequence[tuple[int, int]]]
fragments: Mapping[_StreamGroupKey, Sequence[str]]
def texts(self) -> tuple[str, ...]:
return tuple("".join(self.fragments[group]) for group in self.groups)
def _group_stream_texts(frame_texts: Sequence[Sequence[tuple[_StreamGroupKey, str]]]) -> _StreamTextGroups:
"""Group the per-frame texts by key in first-seen order; ``groups`` keeps only the groups with any text."""
order: Final[list[_StreamGroupKey]] = [] # mutable-ok: built in one pass over the frames
members: Final[dict[_StreamGroupKey, list[tuple[int, int]]]] = {} # mutable-ok: built in one pass over the frames
fragments: Final[dict[_StreamGroupKey, list[str]]] = {} # mutable-ok: built in one pass over the frames
for frame_idx, texts in enumerate(frame_texts):
for local_idx, (group_key, text) in enumerate(texts):
if group_key not in members:
members[group_key] = []
fragments[group_key] = []
order.append(group_key)
members[group_key].append((frame_idx, local_idx))
fragments[group_key].append(text)
return _StreamTextGroups(
groups=tuple(group for group in order if "".join(fragments[group])),
members=members,
fragments=fragments,
)
def _split_group_texts(groups: _StreamTextGroups, rewritten_texts: Sequence[str]) -> Mapping[tuple[int, int], str]:
"""Cut each group's rewritten text back into its fragments, sized like the original fragments."""
split: Final[dict[tuple[int, int], str]] = {} # mutable-ok: built in one pass over the groups
for group_key, rewritten in zip(groups.groups, rewritten_texts):
members = groups.members[group_key]
originals = groups.fragments[group_key]
total_orig = sum(len(t) for t in originals) or 1
pos = 0
for k, member in enumerate(members):
if k == len(members) - 1:
split[member] = rewritten[pos:]
else:
end = pos + round(len(rewritten) * len(originals[k]) / total_orig)
split[member] = rewritten[pos:end]
pos = end
return split
def _event_stream_message(headers_bytes: bytes, payload: bytes) -> bytes:
"""One AWS event-stream message: prelude, prelude CRC, headers, payload and message CRC."""
prelude: Final = struct.pack("!II", 12 + len(headers_bytes) + len(payload) + 4, len(headers_bytes))
prelude_crc: Final = crc32(prelude) & 0xFFFFFFFF
message: Final = prelude + struct.pack("!I", prelude_crc) + headers_bytes + payload
return message + struct.pack("!I", crc32(message) & 0xFFFFFFFF)
def _string_header(name: str, value: str) -> bytes:
name_bytes: Final = name.encode()
value_bytes: Final = value.encode()
return bytes((len(name_bytes),)) + name_bytes + b"\x07" + struct.pack("!H", len(value_bytes)) + value_bytes
def _string_headers(headers: Mapping[str, str]) -> bytes:
return b"".join(_string_header(name, value) for name, value in headers.items())
def _converse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
return _event_stream_message(
_string_headers({":event-type": event_type, ":content-type": "application/json", ":message-type": "event"}),
json.dumps(payload, separators=(",", ":")).encode(),
)
def _converse_exception(message: str, status_code: int) -> bytes:
"""An in-stream exception frame, which boto3 raises as ``EventStreamError`` carrying ``message``."""
return _event_stream_message(
_string_headers(
{
":exception-type": "validationException" if status_code < 500 else "internalServerException",
":content-type": "application/json",
":message-type": "exception",
}
),
json.dumps({"message": message}, separators=(",", ":")).encode(),
)
@dataclass(slots=True)
class ConverseStreamFrame:
"""One frame of a Converse event stream as the proxy relays it.
``payload`` is the parsed JSON of an event frame; exception frames and bytes that do not decode keep it None and
are relayed as received. ``ends_stream`` marks the empty item the framer adds once the upstream has ended, so an
exception frame or a closed connection ends the stream as ``messageStop`` does. ``guardrailed_payload`` is set
when a guardrail rewrote the frame's text."""
raw: bytes
event_type: str | None = None
payload: Mapping[str, object] | None = None
guardrailed_payload: Mapping[str, object] | None = None
ends_stream: bool = False
@property
def block_index(self) -> int | None:
index: Final = self.payload.get("contentBlockIndex") if self.payload is not None else None
return index if isinstance(index, int) else None
def scanned_texts(self) -> tuple[tuple[_StreamGroupKey, str], ...]:
current: Final = self.guardrailed_payload if self.guardrailed_payload is not None else self.payload
return _delta_frame_texts(current) if self.event_type == "contentBlockDelta" and current is not None else ()
def received_texts(self) -> tuple[tuple[_StreamGroupKey, str], ...]:
return (
_delta_frame_texts(self.payload)
if self.event_type == "contentBlockDelta" and self.payload is not None
else ()
)
def encode(self) -> bytes:
if self.guardrailed_payload is None:
return self.raw
headers_length: Final = _unpack_uint32(self.raw[4:8])
return _event_stream_message(
self.raw[12 : 12 + headers_length], json.dumps(self.guardrailed_payload, separators=(",", ":")).encode()
)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
def _decode_converse_frame(raw: bytes) -> ConverseStreamFrame:
from botocore.eventstream import EventStreamBuffer, ParserError
buffer: Final = EventStreamBuffer()
buffer.add_data(raw)
try:
message: Final = next(iter(buffer))
except (ParserError, StopIteration) as e:
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: could not decode event-stream frame, relaying it unmodified: %s", e
)
return ConverseStreamFrame(raw=raw)
# botocore-stubs types EventStreamMessage.headers as int; at runtime it is the decoded header dict.
headers: Final = _JSON_OBJECT.validate_python(message.headers)
event_type: Final = headers.get(":event-type")
if headers.get(":message-type") != "event" or not isinstance(event_type, str):
return ConverseStreamFrame(raw=raw)
try:
payload: Final = _JSON_OBJECT.validate_python(json.loads(message.payload))
except (json.JSONDecodeError, UnicodeDecodeError, ValidationError):
return ConverseStreamFrame(raw=raw, event_type=event_type)
return ConverseStreamFrame(raw=raw, event_type=event_type, payload=payload)
async def _converse_stream_frames(chunks: AsyncIterable[bytes]) -> AsyncGenerator[ConverseStreamFrame, None]:
"""Cut the upstream bytes into frames as soon as each frame has fully arrived, then add an empty item that ends
the stream once the upstream has ended.
Bytes that cannot be framed (a length below the 16-byte minimum, or a stream that ends inside a frame) are relayed
as received."""
pending = b"" # rebind-ok: bytes received but not yet framed
framing = True # rebind-ok: cleared once a frame length is unusable
async for chunk in chunks:
if not framing:
yield ConverseStreamFrame(raw=chunk)
continue
pending += chunk
while len(pending) >= 4:
total_length = _unpack_uint32(pending[:4])
if total_length < 16:
framing = False
break
if len(pending) < total_length:
break
yield _decode_converse_frame(pending[:total_length])
pending = pending[total_length:]
if not framing and pending:
yield ConverseStreamFrame(raw=pending)
pending = b""
if pending:
yield ConverseStreamFrame(raw=pending)
yield ConverseStreamFrame(raw=b"", ends_stream=True)
def _converse_frames(items: Sequence[object]) -> tuple[ConverseStreamFrame, ...]:
"""The frames among ``items``, with the bytes an earlier guardrail's block message added read back into frames."""
return tuple(
item if isinstance(item, ConverseStreamFrame) else _decode_converse_frame(item)
for item in items
if isinstance(item, (ConverseStreamFrame, bytes))
)
def _reads_stream_marks(guardrail: "CustomGuardrail") -> bool:
return getattr(guardrail, "reads_stream_marks", False) is True
def _json_object_member(value: object, key: str) -> Mapping[str, object] | None:
"""``value[key]`` when both are JSON objects, else None."""
try:
return _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_python(value).get(key))
except ValidationError:
return None
def _reject_json_constant(name: str) -> object:
raise ValueError(f"{name} is not JSON")
def _finished_tool_input(text: str) -> object:
"""A tool call's streamed input as the JSON document Converse returns, or the text as received when it is not
JSON (cut short, rewritten by a guardrail, or holding NaN or nesting too deep to send on)."""
try:
return json.loads(text, parse_constant=_reject_json_constant) if text else {}
except (ValueError, RecursionError):
return text
def _finished_converse_content(
frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups
) -> list[dict[str, object]]: # mutable-ok: the content of the finished response guardrails receive
"""The finished reply's text, reasoning and tool-call blocks, shaped as a non-streaming Converse response
holds them."""
texts: Final = dict(zip(groups.groups, groups.texts()))
tool_starts: Final[dict[int | None, Mapping[str, object]]] = {} # mutable-ok: built in one pass over the frames
keys: Final[dict[_StreamGroupKey, None]] = {} # mutable-ok: blocks in stream order, built in one pass
for frame in frames:
tool_use = (
_json_object_member(_json_object_member(frame.payload, "start"), "toolUse")
if frame.event_type == "contentBlockStart"
else None
)
if tool_use is not None:
tool_starts[frame.block_index] = tool_use
keys[(frame.block_index, "tool")] = None
keys.update((key, None) for key, _ in frame.scanned_texts())
content: Final[list[dict[str, object]]] = [] # mutable-ok: built in one pass over the blocks
for block_index, kind in keys:
text = texts.get((block_index, kind), "")
if kind == "tool":
tool_start = tool_starts.get(block_index, {})
named = {name: tool_start[name] for name in ("toolUseId", "name", "type") if name in tool_start}
content.append({"toolUse": {**named, "input": _finished_tool_input(text)}})
elif text and kind == "reasoning":
content.append({"reasoningContent": {"reasoningText": {"text": text}}})
elif text:
content.append({"text": text})
return content
def _stream_marked_request_data(
request_data: Mapping[str, object], frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups, ended: bool
) -> dict[str, object]: # mutable-ok: the request dict guardrails receive
"""``request_data`` with the marks chat completions gives a streamed answer: ``stream`` on every scan and, once
the stream has ended, the finished ``response``."""
if not ended:
return {**request_data, "stream": True}
stop_reason: Final = next(
(
frame.payload.get("stopReason")
for frame in frames
if frame.event_type == "messageStop" and frame.payload is not None
),
None,
)
message: Final = {"role": "assistant", "content": _finished_converse_content(frames, groups)}
response: Final = {"output": {"message": message}, **({"stopReason": stop_reason} if stop_reason else {})}
return {**request_data, "stream": True, "response": response}
def _write_back_stream_texts(
frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups, rewritten_texts: Sequence[str]
) -> None:
split: Final = _split_group_texts(groups, rewritten_texts)
for frame_idx, frame in enumerate(frames):
current = frame.guardrailed_payload if frame.guardrailed_payload is not None else frame.payload
if frame.event_type != "contentBlockDelta" or current is None:
continue
rewritten = copy.deepcopy(current)
changed = False
for local_idx, (_, container, key) in enumerate(_collect_stream_delta_text_holders(rewritten.get("delta"))):
new_text = split.get((frame_idx, local_idx))
if new_text is not None and new_text != container[key]:
container[key] = new_text
changed = True
if changed:
frame.guardrailed_payload = rewritten
class BedrockPassthroughGuardrailHandler(BaseTranslation):
@staticmethod
def is_event_stream_content_type(content_type: str) -> bool:
@ -640,10 +945,6 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
user_api_key_dict: "UserAPIKeyAuth",
data: dict,
) -> bytes:
import json as _json
import struct
from binascii import crc32 as esm_crc32
from botocore.eventstream import EventStreamBuffer
frames: Final[list[_StreamFrame]] = []
@ -671,16 +972,11 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
frames.append({"raw": frame_raw, "texts": []})
continue
texts: list[tuple[_StreamGroupKey, str]] = []
texts: Sequence[tuple[_StreamGroupKey, str]] = ()
if event_type == "contentBlockDelta":
try:
payload_dict: dict[str, object] = _json.loads(payload_bytes)
block_index = payload_dict.get("contentBlockIndex")
# Group per content block: a guardrail edit that changes length must not move text between blocks.
texts = [
((block_index if isinstance(block_index, int) else None, group_key), container[key])
for group_key, container, key in _collect_stream_delta_text_holders(payload_dict.get("delta"))
]
payload_dict: dict[str, object] = json.loads(payload_bytes)
texts = _delta_frame_texts(payload_dict)
except Exception as e:
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: could not parse "
@ -692,27 +988,15 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
trailing_bytes: Final = body_bytes[offset:]
group_order: Final[list[_StreamGroupKey]] = []
group_members: Final[dict[_StreamGroupKey, list[tuple[int, int]]]] = {}
group_texts: Final[dict[_StreamGroupKey, list[str]]] = {}
for frame_idx, frame in enumerate(frames):
for local_idx, (group_key, text) in enumerate(frame["texts"]):
if group_key not in group_members:
group_members[group_key] = []
group_texts[group_key] = []
group_order.append(group_key)
group_members[group_key].append((frame_idx, local_idx))
group_texts[group_key].append(text)
active_groups: Final = [gk for gk in group_order if "".join(group_texts[gk])]
if not active_groups:
groups: Final = _group_stream_texts([frame["texts"] for frame in frames])
if not groups.groups:
return body_bytes
synthetic_response: Final[dict] = {
"output": {
"message": {
"role": "assistant",
"content": [{"text": "".join(group_texts[gk])} for gk in active_groups],
"content": [{"text": text} for text in groups.texts()],
}
},
"stopReason": "end_turn",
@ -734,24 +1018,11 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
try:
processed_blocks: Final = processed["output"]["message"]["content"]
de_anonymized_texts: Final = [processed_blocks[i]["text"] for i in range(len(active_groups))]
de_anonymized_texts: Final = [processed_blocks[i]["text"] for i in range(len(groups.groups))]
except (KeyError, IndexError, TypeError):
return body_bytes
new_text_map: Final[dict[tuple[int, int], str]] = {}
for group_key, de_anonymized_text in zip(active_groups, de_anonymized_texts):
members = group_members[group_key]
orig_texts = group_texts[group_key]
total_orig = sum(len(t) for t in orig_texts) or 1
de_anon_len = len(de_anonymized_text)
pos = 0
for k, member in enumerate(members):
if k == len(members) - 1:
new_text_map[member] = de_anonymized_text[pos:]
else:
end = pos + round(de_anon_len * len(orig_texts[k]) / total_orig)
new_text_map[member] = de_anonymized_text[pos:end]
pos = end
new_text_map: Final = _split_group_texts(groups, de_anonymized_texts) # pyright: ignore[reportUnknownArgumentType] # the hook returns an untyped response
result_parts: Final[list[bytes]] = []
@ -766,27 +1037,19 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
headers_bytes = frame_raw[12 : 12 + orig_hdrs_len]
try:
payload_dict = _json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
for local_idx, (_, container, key) in enumerate(
_collect_stream_delta_text_holders(payload_dict.get("delta"))
):
new_text = new_text_map.get((frame_idx, local_idx))
if new_text is not None:
container[key] = new_text
new_payload = _json.dumps(payload_dict, separators=(",", ":")).encode()
new_payload = json.dumps(payload_dict, separators=(",", ":")).encode()
except Exception:
result_parts.append(frame_raw)
continue
new_total = 12 + orig_hdrs_len + len(new_payload) + 4
prelude = struct.pack("!II", new_total, orig_hdrs_len)
prelude_crc_val = esm_crc32(prelude) & 0xFFFFFFFF
prelude_crc_b = struct.pack("!I", prelude_crc_val)
part_for_msg_crc = prelude_crc_b + headers_bytes + new_payload
msg_crc_val = esm_crc32(part_for_msg_crc, prelude_crc_val) & 0xFFFFFFFF
msg_crc_b = struct.pack("!I", msg_crc_val)
result_parts.append(prelude + prelude_crc_b + headers_bytes + new_payload + msg_crc_b)
result_parts.append(_event_stream_message(headers_bytes, new_payload))
result_parts.append(trailing_bytes)
return b"".join(result_parts)
@ -948,6 +1211,20 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
if not texts:
return response
await self._scan_output_texts(
texts, holders, guardrail_to_apply, litellm_logging_obj, user_api_key_dict, request_data
)
return response
async def _scan_output_texts(
self,
texts: list[str],
holders: list[_StringHolder],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"],
user_api_key_dict: Optional["UserAPIKeyAuth"],
request_data: dict[str, object] | None, # mutable-ok: the request dict guardrails receive
) -> None:
effective_request_data = request_data or {}
if "litellm_metadata" not in effective_request_data and user_api_key_dict is not None:
user_metadata: Final = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
@ -958,8 +1235,8 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
}
inputs: Final = GenericGuardrailAPIInputs(texts=texts)
model: Final = effective_request_data.get("model") if effective_request_data else None
if model:
model: Final = effective_request_data.get("model")
if isinstance(model, str) and model:
inputs["model"] = model
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
@ -973,4 +1250,122 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
if guardrailed_texts:
_write_back_texts(guardrailed_texts, holders)
return response
async def process_output_streaming_response(
self,
responses_so_far: Sequence[object],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
request_data: dict[str, object] | None = None, # mutable-ok: the request dict guardrails receive
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> Sequence[object]:
"""
Scan the Converse frames relayed so far as the buffered path scans a whole response: one text per content
block and kind. Once the stream has ended, guardrail rewrites are written back into the frames, which reach
the caller only when the stream is held until moderated. A guardrail that sets ``reads_stream_marks`` gets
a copy of the request marked as chat completions marks a streamed answer; the others get it unchanged.
Such a guardrail also gets the ended scan of a reply with no text, such as a call to a tool that takes no
input, with no texts to scan.
"""
frames: Final = _converse_frames(responses_so_far)
groups: Final = _group_stream_texts([frame.scanned_texts() for frame in frames])
ended: Final = self._check_streaming_has_ended(frames)
reads_marks: Final = _reads_stream_marks(guardrail_to_apply)
if not groups.groups:
if request_data is not None and reads_marks and ended and _finished_converse_content(frames, groups):
await self._scan_output_texts(
[],
[],
guardrail_to_apply,
litellm_logging_obj,
user_api_key_dict,
_stream_marked_request_data(request_data, frames, groups, ended),
)
return responses_so_far
content: Final = [{"text": text} for text in groups.texts()]
await self.process_output_response( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
response={"output": {"message": {"role": "assistant", "content": content}}},
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=(
_stream_marked_request_data(request_data, frames, groups, ended)
if request_data is not None and reads_marks
else request_data
),
)
if ended:
_write_back_stream_texts(frames, groups, [block["text"] for block in content])
return responses_so_far
def needs_ended_stream_scan(self, guardrail: "CustomGuardrail", responses_so_far: Sequence[object]) -> bool:
return _reads_stream_marks(guardrail)
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
frames: Final = _converse_frames(responses_so_far)
return StreamingScanKey(
texts=_group_stream_texts([frame.received_texts() for frame in frames]).texts(),
stream_ended=self._check_streaming_has_ended(frames),
)
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
return any(
frame.event_type == "messageStop" or frame.ends_stream for frame in _converse_frames(responses_so_far)
)
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes]:
"""
End the message with the guardrail's message as a new text block and ``stopReason`` ``guardrail_intervened``,
closing any content block left open and starting the message when nothing was relayed yet.
"""
frames: Final = _converse_frames(responses_so_far or ())
started: Final = frozenset(
index
for index in (
frame.block_index for frame in frames if frame.event_type in ("contentBlockStart", "contentBlockDelta")
)
if index is not None
)
stopped: Final = frozenset(
index
for index in (frame.block_index for frame in frames if frame.event_type == "contentBlockStop")
if index is not None
)
block_index: Final = max(started | stopped, default=-1) + 1
return (
*(
()
if any(frame.event_type == "messageStart" for frame in frames)
else (_converse_event("messageStart", {"role": "assistant"}),)
),
*(_converse_event("contentBlockStop", {"contentBlockIndex": index}) for index in sorted(started - stopped)),
_converse_event("contentBlockDelta", {"contentBlockIndex": block_index, "delta": {"text": exc.message}}),
_converse_event("contentBlockStop", {"contentBlockIndex": block_index}),
_converse_event("messageStop", {"stopReason": "guardrail_intervened"}),
)
@staticmethod
def owns_event_stream_item(item: object) -> bool:
return isinstance(item, ConverseStreamFrame)
@staticmethod
def event_stream_items(chunks: AsyncIterable[bytes]) -> AsyncGenerator[ConverseStreamFrame, None]:
return _converse_stream_frames(chunks)
@staticmethod
def encode_event_stream_item(item: object) -> bytes:
if isinstance(item, ConverseStreamFrame):
return item.encode()
if isinstance(item, bytes):
return item
raise TypeError(f"cannot relay {type(item).__name__} on a Converse event stream")
@staticmethod
def event_stream_error_item(message: str, status_code: int) -> bytes:
return _converse_exception(message, status_code)

65
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -1,14 +1,18 @@
import json
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from functools import partial
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Optional, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
from litellm.types.utils import ModelResponseStream
from ..base_aws_llm import BaseAWSLLM
@ -91,6 +95,57 @@ def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelR
return None
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_JSON_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def _leaf_texts(block: Mapping[str, object]) -> Iterator[str]:
"""The prompt text a content block carries itself: its text or JSON, or a tool call's input."""
text: Final = block.get("text")
if isinstance(text, str):
yield text
if "json" in block:
yield json.dumps(block["json"], default=str)
if "toolUse" in block:
yield json.dumps(_JSON_OBJECT.validate_python(block["toolUse"]).get("input"), default=str)
def _block_texts(block: Mapping[str, object]) -> Iterator[str]:
"""The prompt text a request content block carries, including the blocks of a tool result's content, which
Bedrock does not nest further."""
yield from _leaf_texts(block)
if "toolResult" not in block:
return
tool_result: Final = _JSON_OBJECT.validate_python(block["toolResult"])
yield from chain.from_iterable(map(_leaf_texts, _JSON_OBJECTS.validate_python(tool_result.get("content", ()))))
def _content_text(content: object) -> str:
if isinstance(content, str):
return content
return "\n".join(chain.from_iterable(map(_block_texts, _JSON_OBJECTS.validate_python(content))))
def _chat_message(turn: Mapping[str, object]) -> ChatCompletionUserMessage:
return ChatCompletionUserMessage(role="user", content=_content_text(turn.get("content", "")))
def _count_prompt_tokens(model: str, request_body: object) -> int:
"""Estimate a Bedrock request's prompt tokens from its system prompt and turns, the way a chat completions
stream that ended before the provider reported usage is estimated."""
from litellm.litellm_core_utils.token_counter import token_counter
body: Final = _JSON_OBJECT.validate_python(request_body)
if "messages" not in body:
return 0
system: Final = body.get("system")
messages: Final = (
*((ChatCompletionSystemMessage(role="system", content=_content_text(system)),) if system else ()),
*map(_chat_message, _JSON_OBJECTS.validate_python(body["messages"])),
)
return token_counter(model=model, messages=messages)
def _build_logged_response(
chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj"
) -> Optional["CostResponseTypes"]:
@ -98,7 +153,13 @@ def _build_logged_response(
if len(chunks) == 0:
return None
return stream_chunk_builder(chunks=list(chunks), logging_obj=litellm_logging_obj)
return stream_chunk_builder(
chunks=list(chunks),
logging_obj=litellm_logging_obj,
count_prompt_tokens=partial(
_count_prompt_tokens, litellm_logging_obj.model, litellm_logging_obj.model_call_details.get("input")
),
)
class BedrockEventStreamCollector:

Some files were not shown because too many files have changed in this diff Show more