scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 233

Merge remote-tracking branch 'origin/main' into litellm_langfuse_otel_metadata_keys

Browse source
This commit is contained in:
yucheng 2026-10-17 05:22:15 -07:00
commit c15dadfd70
606 changed files with 57980 additions and 6031 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
.github/scripts/assert_ci_coverage.py vendored
View file

@ -32,6 +32,7 @@ IGNORE_ARG_RE: Final = re.compile(r"--ignore(?:-glob)?[= ](\S+)")
DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
DEDICATED_INTEGRATION_WORKFLOWS: Final = MappingProxyType({"image-scan.yml": "tests/integration/database/image/"})
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
# tests has to be named by some shard or it runs nowhere. A child listed here is
@ -560,6 +561,17 @@ def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
return frozenset(ast.literal_eval(values))
def _owns(workflow: str, relative_path: str) -> bool:
owned_root: Final = DEDICATED_INTEGRATION_WORKFLOWS.get(workflow)
return owned_root is not None and relative_path.startswith(owned_root)
def _tokens_outside_owner(relative_path: str, workflow_tokens: Mapping[str, frozenset[str]]) -> frozenset[str]:
return frozenset[str]().union(
*(tokens for workflow, tokens in workflow_tokens.items() if not _owns(workflow, relative_path))
)
def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
runner: Final = repo_root / "tests/integration/run.py"
if not runner.exists():
@ -611,15 +623,20 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
)
!= 1
)
gha_tokens: Final = _invoked_test_tokens(
scalar
for path in (repo_root / ".github/workflows").glob("*.y*ml")
for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
workflow_tokens: Final = MappingProxyType(
{
path.name: _invoked_test_tokens(_scalars(yaml.safe_load(path.read_text()), path.name))
for path in (repo_root / ".github/workflows").glob("*.y*ml")
}
)
gha_tokens: Final = frozenset[str]().union(*workflow_tokens.values())
circle_owned: Final = (
frozenset(str(path.relative_to(repo_root)) for path in integration_root.rglob("test_*.py")) - github_files
)
findings: Final = (
tuple(
Finding(path, "integration contract is also selected by GitHub Actions")
for path in paths
Finding(path, "integration file outside GITHUB_FILES is selected by GitHub Actions")
for path in sorted(circle_owned)
if any(_token_covers(token, path) for token in gha_tokens)
)
+ tuple(
@ -632,6 +649,11 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
for path in sorted(github_files)
if not (repo_root / path).is_file()
)
+ tuple(
Finding(path, "GitHub-owned integration contract runs outside a dedicated integration workflow")
for path in sorted(github_files)
if any(_token_covers(token, path) for token in _tokens_outside_owner(path, workflow_tokens))
)
)
browser_commands: Final = tuple(
scalar.value
@ -688,13 +710,9 @@ def main() -> int:
scalars = _all_scalars()
integration_paths, ownership_findings = _integration_ownership()
test_findings = (
_uncovered_tests(
allowlist,
_invoked_selections(scalars)
+ (Selection(included=integration_paths, ignored=frozenset()),),
)
+ ownership_findings
test_findings = _uncovered_tests(
allowlist,
_invoked_selections(scalars) + (Selection(included=integration_paths, ignored=frozenset()),),
)
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
@ -711,13 +729,25 @@ def main() -> int:
test_findings,
"Add each to a job's test path, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if ownership_findings:
browser_remedy = (
" CircleCI's browser group owns the integrationCritical Playwright contracts, so no GitHub workflow may "
"select them and the default Playwright config must exclude them."
if any("browser" in finding.detail or "Playwright" in finding.detail for finding in ownership_findings)
else ""
)
_report(
"integration contracts with the wrong CI owner",
ownership_findings,
"Follow tests/integration/AGENTS.md, which says which CI job owns each integration file." + browser_remedy,
)
if dockerfile_findings:
_report(
"Dockerfiles that no CI job builds",
dockerfile_findings,
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if stale_findings or test_findings or dockerfile_findings:
if stale_findings or test_findings or ownership_findings or dockerfile_findings:
return 1
_write(

5
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -24,12 +24,9 @@ jobs:
- name: Update JSON Data
run: |
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
- name: Regenerate JSON Schema
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create Pull Request
run: |
git add model_prices_and_context_window.json model_prices_and_context_window.schema.json
git add model_prices_and_context_window.json
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
gh pr create --title "Update model_prices_and_context_window.json file" \
--body "Automated update for model_prices_and_context_window.json" \

7
.github/workflows/sync-together-ai-models.yml vendored
View file

@ -41,10 +41,6 @@ jobs:
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Regenerate the JSON schema
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
@ -57,8 +53,7 @@ jobs:
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json \
model_prices_and_context_window.schema.json
litellm/model_prices_and_context_window_backup.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"

8
.github/workflows/test-rust.yml vendored
View file

@ -16,6 +16,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"
@ -46,6 +50,10 @@ on:
- "litellm/proxy/guardrails/**"
- "litellm/utils.py"
- "litellm/ocr/**"
- "litellm/chat_completions/dispatch.py"
- "litellm/embeddings/dispatch.py"
- "litellm/messages/dispatch.py"
- "litellm/responses/dispatch.py"
- "litellm/llms/base_llm/ocr/**"
- "litellm/llms/custom_httpx/llm_http_handler.py"
- "Makefile"

1
.github/workflows/test-unit.yml vendored
View file

@ -696,7 +696,6 @@ jobs:
tests/unit/responses/litellm_completion_transformation
tests/unit/responses/test_additional_tools.py
tests/unit/responses/test_custom_tool_call.py
tests/unit/responses/test_dispatch.py
tests/unit/responses/test_metadata_codex_callback.py
tests/unit/responses/test_no_duplicate_spend_logs.py
tests/unit/responses/test_null_test_fix.py

4
AGENTS.md
View file

@ -29,7 +29,7 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
`tests/unit/` mirrors `litellm/` in a parallel path (see `tests/unit/AGENTS.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `AGENTS.md`
End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
@ -72,7 +72,7 @@ When referencing or running models (coding, QA'ing, writing docs, writing tests,
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off the repository's current default branch and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Internal contributors branch a new PR off the repository's current default branch, with a branch name prefixed with litellm_
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch

1
README.md
View file

@ -384,6 +384,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | |
| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | |
| [Nace (`nace`)](https://console.nace.ai/docs/drex) | | | | | | | | | | |
| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |
| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | | | | | | | |
| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ | | | | | | | |

32
backend/Dockerfile
View file

@ -4,6 +4,33 @@ ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2
FROM $UV_IMAGE AS uvbin
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -54,7 +81,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -63,6 +90,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \

45
ci_cd/cost_map_guard.py
View file

@ -1,11 +1,11 @@
"""Guard the cost map on pull requests.
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
files and may only add or update models.
Every pull request whose diff against its merge base touches one of the two cost map files gets the file
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
get the file checks and additionally may only touch those two files or the legacy schema
path and may only add or update models.
"""
from __future__ import annotations
@ -19,13 +19,12 @@ from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
CostMap = dict[str, object]
@ -35,7 +34,6 @@ CostMap = dict[str, object]
class Snapshot:
cost_map: str
backup: str
schema: str
def _parse_object(text: str, path: str) -> CostMap | str:
@ -46,13 +44,6 @@ def _parse_object(text: str, path: str) -> CostMap | str:
return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
def _rendered_schema(cost_map: CostMap) -> str:
try:
return render(build_schema(cost_map))
except SystemExit as error:
return str(error)
def _evaluation_output_price_failure(
key: str, entry: object, evaluation_output_price_sources: Mapping[str, str]
) -> str | None:
@ -78,29 +69,13 @@ def _file_failures(
*,
evaluation_output_price_sources: Mapping[str, str] = EVALUATION_OUTPUT_PRICE_SOURCES,
) -> tuple[str, ...]:
schema_text: Final = _rendered_schema(head_map)
if not schema_text.startswith("{"):
return (schema_text,)
backup_failure: Final = (
()
if head.backup == head.cost_map
else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
)
schema_failure: Final = (
()
if head.schema == schema_text
else (
f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
"run `python ci_cd/generate_model_prices_schema.py` and commit the result",
)
)
return (
*backup_failure,
*schema_failure,
*(
f"{COST_MAP_PATH} does not validate against its schema: {error}"
for error in validation_errors(head_map, json.loads(schema_text))[:20]
),
*(
failure
for key, entry in head_map.items()
@ -135,7 +110,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
*(
f"bot PRs may only change the cost map files, not {path}"
for path in changed_files
if path not in GUARDED_PATHS
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
),
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),

418
ci_cd/generate_model_prices_schema.py
View file

@ -1,418 +0,0 @@
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Final, Optional
import jsonschema
REPO_ROOT = Path(__file__).parent.parent
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
SCHEMA_PATH = REPO_ROOT / "model_prices_and_context_window.schema.json"
SPECIAL_ROOT_KEYS = frozenset({"sample_spec", "fallback_generalizations"})
JsonSchema = dict
NONNEG_NUMBER: JsonSchema = {"type": "number", "minimum": 0}
NONNEG_INTEGER: JsonSchema = {"type": "integer", "minimum": 0}
BOOLEAN: JsonSchema = {"type": "boolean"}
STRING: JsonSchema = {"type": "string"}
TIME_WINDOW: Final[JsonSchema] = {"type": "string", "pattern": r"^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$"}
WEEKDAY_PATTERN: Final = (
r"(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$"
)
EXTRA_BOOLEAN_KEYS = frozenset(
{
"gemini_native_audio",
"gemini_audio_only_live",
"uses_embed_content",
"use_openai_responses_path",
"bedrock_converse_supports_strict_tools",
"thinking_always_on",
}
)
HOURS_UTC: Final[JsonSchema] = {
"description": 'UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.',
"oneOf": [TIME_WINDOW, {"type": "array", "items": TIME_WINDOW, "minItems": 1}],
}
OFF_PEAK_WINDOW: Final[JsonSchema] = {
"type": "object",
"properties": {
"hours_utc": HOURS_UTC,
"weekdays": {
"type": "array",
"description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.",
"items": {
"oneOf": [
{"type": "integer", "minimum": 1, "maximum": 7},
{"type": "string", "pattern": WEEKDAY_PATTERN},
]
},
"minItems": 1,
},
},
"required": ["hours_utc"],
"additionalProperties": False,
}
OBJECT_KEYS: dict[str, JsonSchema] = {
"off_peak_pricing": {
"type": "object",
"description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.",
"properties": {
"hours_utc": HOURS_UTC,
"windows": {"type": "array", "items": OFF_PEAK_WINDOW, "minItems": 1},
"weekday_timezone": {
"type": "string",
"description": "IANA zone the weekdays of each window are read on; defaults to UTC.",
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
},
"anyOf": [{"required": ["hours_utc"]}, {"required": ["windows"]}],
"additionalProperties": False,
},
"search_context_cost_per_query": {
"type": "object",
"description": "USD cost per web search query, keyed by search context size.",
"properties": {
"search_context_size_low": NONNEG_NUMBER,
"search_context_size_medium": NONNEG_NUMBER,
"search_context_size_high": NONNEG_NUMBER,
},
"additionalProperties": False,
},
"guardrail_cost_per_unit": {
"type": "object",
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
"additionalProperties": NONNEG_NUMBER,
},
"metadata": {
"type": "object",
"description": "Free-form notes about the entry (e.g. pricing derivation).",
},
"provider_specific_entry": {
"type": "object",
"description": "Provider-internal routing hints (e.g. bedrock_invocation_schema).",
},
}
ARRAY_KEYS: dict[str, JsonSchema] = {
"supported_audio_formats": {
"type": "array",
"description": "Audio container formats the model can return.",
"items": {"type": "string", "enum": ["mp3", "wav"]},
},
"supported_endpoints": {
"type": "array",
"description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.",
"items": STRING,
},
"supported_modalities": {
"type": "array",
"description": "Input modalities the model accepts.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video"]},
},
"supported_output_modalities": {
"type": "array",
"description": "Output modalities the model can produce.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
},
"reasoning_effort_levels": {
"type": "array",
"description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
"items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
},
"supported_regions": {
"type": "array",
"description": "Cloud regions the model is available in ('global' or region ids).",
"items": STRING,
},
"tiered_pricing": {
"type": "array",
"description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",
"items": {
"type": "object",
"properties": {
"range": {
"type": "array",
"description": "[min, max] prompt-token span this tier applies to.",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"max_results_range": {
"type": "array",
"description": "[min, max] result-count span this tier applies to (search models).",
"items": NONNEG_NUMBER,
"minItems": 2,
"maxItems": 2,
},
"input_cost_per_token": NONNEG_NUMBER,
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
"input_cost_per_query": NONNEG_NUMBER,
},
"additionalProperties": False,
},
},
}
INTEGER_KEYS: dict[str, JsonSchema] = {
"max_tokens": {
**NONNEG_INTEGER,
"description": "Legacy field: max output tokens if the provider specifies it, else max input tokens.",
},
"max_input_tokens": {
**NONNEG_INTEGER,
"description": "Maximum prompt/context tokens the model accepts.",
},
"max_output_tokens": {
**NONNEG_INTEGER,
"description": "Maximum tokens the model can generate in one response.",
},
"output_vector_size": {
**NONNEG_INTEGER,
"description": "Embedding dimension for embedding models.",
},
"prompt_cache_min_tokens": {
**NONNEG_INTEGER,
"description": "Smallest prefix the provider will actually cache; absent means the provider default applies.",
},
"token_count_framing": {
**NONNEG_INTEGER,
"description": "Tokens added once per request to a local count estimate, for the provider's chat framing.",
},
"tpm": {**NONNEG_INTEGER, "description": "Provider default tokens-per-minute limit."},
"rpm": {**NONNEG_INTEGER, "description": "Provider default requests-per-minute limit."},
}
NUMBER_KEYS: dict[str, JsonSchema] = {
"regional_processing_uplift_multiplier_eu": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).",
},
"regional_processing_uplift_multiplier_us": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
},
"regional_endpoint_uplift_multiplier": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
},
}
COST_DESCRIPTIONS: dict[str, str] = {
"input_cost_per_token": "USD per prompt token.",
"output_cost_per_token": "USD per generated token.",
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
"google_maps_grounding_cost_per_query": (
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
}
def cost_description(key: str) -> Optional[str]:
if key in COST_DESCRIPTIONS:
return COST_DESCRIPTIONS[key]
if key.endswith("_flex"):
return "Flex service-tier rate for the same-named base field."
if key.endswith("_priority"):
return "Priority service-tier rate for the same-named base field."
if "_above_" in key:
return "Rate applied once the prompt exceeds the token threshold in the field name."
return None
def cost_schema(key: str) -> JsonSchema:
description = cost_description(key)
return {**NONNEG_NUMBER, "description": description} if description else dict(NONNEG_NUMBER)
def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
return {
"litellm_provider": {
"type": "string",
"description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.",
},
"mode": {
"type": "string",
"description": "Primary API surface / task type of the model.",
"enum": list(modes),
},
"source": {
"type": "string",
"description": "URL of the provider pricing/model page this entry was taken from.",
},
"deprecation_date": {
"type": "string",
"description": "Date the provider deprecates the model, YYYY-MM-DD.",
"format": "date",
"pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$",
},
"web_search_billing_unit": {
"type": "string",
"description": "Whether web search is billed per query or per prompt.",
"enum": ["per_query", "per_prompt"],
},
"bedrock_output_config_effort_ceiling": {
"type": "string",
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"token_count_fallback_model": {
"type": "string",
"minLength": 1,
"description": "Model that counts tokens for this one when the provider cannot count it.",
},
"audio_transcription_config": STRING,
"vertex_ai_audio_api": {
"type": "string",
"enum": ["lyria_predict", "lyria_interactions"],
},
}
def classify(key: str, modes: tuple) -> Optional[JsonSchema]:
curated = {**OBJECT_KEYS, **ARRAY_KEYS, **string_key_schemas(modes), **INTEGER_KEYS, **NUMBER_KEYS}
if key in curated:
return curated[key]
if key.startswith("supports_") or key in EXTRA_BOOLEAN_KEYS:
return BOOLEAN
if "cost" in key:
return cost_schema(key)
return None
def build_schema(prices: dict) -> JsonSchema:
entries = {name: entry for name, entry in prices.items() if name not in SPECIAL_ROOT_KEYS}
all_keys = tuple(sorted({key for entry in entries.values() for key in entry}))
modes = tuple(sorted({entry["mode"] for entry in entries.values() if "mode" in entry}))
unclassified = tuple(key for key in all_keys if classify(key, modes) is None)
if unclassified:
raise SystemExit(
f"Unclassified keys in {PRICES_PATH.name}: {', '.join(unclassified)}. "
f"Add them to the key tables in {Path(__file__).name} and rerun it."
)
entry_properties = {key: classify(key, modes) for key in all_keys}
return {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "LiteLLM model_prices_and_context_window.json",
"description": (
"Schema for LiteLLM's model price and context window registry "
"(https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). "
"Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, "
"optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. "
"All costs are USD per unit. New optional fields are added regularly, so consumers should "
"ignore unknown fields rather than reject them."
),
"type": "object",
"properties": {
"sample_spec": {
"type": "object",
"description": (
"Documentation placeholder illustrating the entry shape; not a real model and not "
"schema-conformant (several values are prose)."
),
},
"fallback_generalizations": {
"type": "object",
"description": "Regex rules that generalize unknown model ids to known families; not a model entry.",
"properties": {
"rules": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": STRING,
"pattern": STRING,
"description": STRING,
},
"required": ["name", "pattern"],
"additionalProperties": True,
},
}
},
"additionalProperties": False,
},
},
"additionalProperties": {"$ref": "#/$defs/modelEntry"},
"$defs": {
"modelEntry": {
"type": "object",
"description": (
"Pricing, limits, and capability flags for one model. Fields other than litellm_provider "
"are optional; boolean capability flags are simply omitted when unknown or false."
),
"required": ["litellm_provider"],
"properties": entry_properties,
"additionalProperties": True,
}
},
}
def render(schema: JsonSchema) -> str:
return json.dumps(schema, indent=2) + "\n"
def validation_errors(prices: dict, schema: JsonSchema) -> tuple:
validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER)
return tuple(
f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}"
for error in validator.iter_errors(prices)
)
def main() -> int:
check = "--check" in sys.argv[1:]
prices = json.loads(PRICES_PATH.read_text())
rendered = render(build_schema(prices))
errors = validation_errors(prices, json.loads(rendered))
if errors:
print(f"{PRICES_PATH.name} does not validate against the generated schema:")
print("\n".join(errors[:20]))
return 1
if not check:
SCHEMA_PATH.write_text(rendered)
print(f"wrote {SCHEMA_PATH}")
return 0
if not SCHEMA_PATH.exists() or SCHEMA_PATH.read_text() != rendered:
print(
f"{SCHEMA_PATH.name} is out of sync with {PRICES_PATH.name}. "
f"Run `python {Path(__file__).relative_to(REPO_ROOT)}` and commit the result."
)
return 1
print(f"{SCHEMA_PATH.name} is in sync and {PRICES_PATH.name} validates against it")
return 0
if __name__ == "__main__":
sys.exit(main())

34
gateway/Dockerfile
View file

@ -20,6 +20,33 @@ RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${
make -j"$(nproc)" pgbouncer && \
install -m 0755 pgbouncer /usr/local/bin/pgbouncer
# ---------- Rust extension ----------
# Copies only the native build's inputs, so the fat-LTO compile stays cached across Python-only changes
FROM $LITELLM_BUILD_IMAGE AS rust-native
WORKDIR /app
USER root
COPY --from=uvbin /uv /uvx /usr/local/bin/
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done
ENV UV_PYTHON_DOWNLOADS=0
COPY pyproject.toml README.md rust-toolchain.toml ./
COPY .cargo ./.cargo
COPY litellm-rust ./litellm-rust
RUN --mount=type=cache,target=/root/.cache/uv \
mkdir -p litellm/rust_bridge /wheels && \
touch litellm/__init__.py litellm/rust_bridge/__init__.py && \
uvx --python python3.13 --from maturin==1.15.0 --with puccinialin==0.1.14 \
python -c "import maturin; maturin.build_wheel('/wheels')"
# ---------- Builder ----------
FROM $LITELLM_BUILD_IMAGE AS builder
@ -66,11 +93,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
# Stage 2 — copy source and install the project + workspace members.
# Stage 2 — copy source and install the workspace members.
COPY . .
RUN --mount=type=cache,target=/root/.cache/uv \
uv sync --frozen --no-default-groups --no-editable \
uv sync --frozen --no-install-project --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--group admin-mcp \
@ -79,6 +106,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra bedrock-realtime \
--python python3.13
RUN --mount=type=bind,from=rust-native,source=/wheels,target=/wheels \
uv pip install --python /app/.venv/bin/python --no-deps /wheels/litellm-*.whl
# PYTHONPATH=/app makes the source tree shadow the installed package, so the
# compiled Rust extension must live next to the source or it is never imported.
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/

23
litellm-proxy-extras/litellm_proxy_extras/migrations/20261015120000_add_environment_variables_revision_trigger/migration.sql Normal file
View file

@ -0,0 +1,23 @@
SET LOCAL lock_timeout = '10s';
CREATE SEQUENCE IF NOT EXISTS "litellm_environment_variables_revision_seq";
GRANT USAGE ON SEQUENCE "litellm_environment_variables_revision_seq" TO PUBLIC;
CREATE OR REPLACE FUNCTION litellm_bump_environment_variables_revision() RETURNS TRIGGER AS $$
DECLARE
next_revision BIGINT := nextval(format('%I.litellm_environment_variables_revision_seq', TG_TABLE_SCHEMA)::regclass);
BEGIN
IF TG_OP = 'UPDATE' THEN
NEW."reload_revision" := GREATEST(next_revision, OLD."reload_revision" + 1);
ELSE
NEW."reload_revision" := next_revision;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS "litellm_environment_variables_revision" ON "LiteLLM_Config";
CREATE TRIGGER "litellm_environment_variables_revision"
BEFORE INSERT OR UPDATE ON "LiteLLM_Config"
FOR EACH ROW WHEN (NEW."param_name" = 'environment_variables')
EXECUTE PROCEDURE litellm_bump_environment_variables_revision();

2
litellm-rust/crates/gateway-inference/src/messages.rs
View file

@ -24,7 +24,7 @@ const OPENROUTER_API_HEADERS: [&str; 2] = [
"x-openrouter-experimental-metadata",
];
const ANTHROPIC_API_HEADER_PROVIDERS: &str =
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter";
"anthropic,bedrock,bedrock_mantle,vertex_ai,openrouter,deepseek";
pub async fn create(
State(gateway): State<Arc<Gateway>>,

2
litellm-rust/crates/gateway-inference/tests/messages.rs
View file

@ -16,7 +16,7 @@ use wiremock::{
#[rstest]
#[case::anthropic("anthropic/test-model", "/v1/messages", true)]
#[case::azure("azure_ai/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", false)]
#[case::deepseek("deepseek/test-model", "/anthropic/v1/messages", true)]
#[tokio::test]
async fn messages_reaches_the_provider_and_preserves_json_or_sse(
#[case] model: &str,

1
litellm-rust/crates/model-catalog/AGENTS.md
View file

@ -2,5 +2,4 @@
For `model_prices_and_context_window.json` validation, we should eventually:
- Remove any schema file like `model_prices_and_context_window.schema.json`
- Stop skipping this crate's tests

37
litellm/__init__.py
View file

@ -1,34 +1,13 @@
### Hide pydantic namespace conflict warnings globally ###
from __future__ import annotations
import warnings
from litellm._process_defaults import apply_process_defaults as _apply_process_defaults
_apply_process_defaults()
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
### INIT VARIABLES #########################
import threading
import os
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
_dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
from collections.abc import Mapping, Sequence
from typing import (
Any,
@ -207,7 +186,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]]
[]
) # internal variable - async custom callbacks are routed here.
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
post_call_rules: List[Callable[..., object]] = []
turn_off_message_logging: Optional[bool] = False
standard_logging_payload_excluded_fields: Optional[List[str]] = (
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
@ -1177,7 +1156,9 @@ def _build_models_by_provider() -> dict:
"edenai": edenai_models,
"datarobot": datarobot_models,
"vertex_ai": vertex_chat_models
| vertex_code_chat_models
| vertex_text_models
| vertex_code_text_models
| vertex_anthropic_models
| vertex_vision_models
| vertex_language_models
@ -1185,7 +1166,7 @@ def _build_models_by_provider() -> dict:
| vertex_minimax_models
| vertex_moonshot_models
| vertex_zai_models,
"ai21": ai21_models,
"ai21": ai21_models | ai21_chat_models,
"bedrock": bedrock_models | bedrock_converse_models,
"petals": petals_models,
"ollama": ollama_models,
@ -1416,6 +1397,7 @@ from .exceptions import (
InvalidRequestError,
BadRequestError,
ImageFetchError,
MalformedRequestFieldError as MalformedRequestFieldError,
VectorStoreSearchError,
NotFoundError,
PermissionDeniedError,
@ -1691,6 +1673,9 @@ if TYPE_CHECKING:
from .llms.perplexity.decisions.transformation import (
PerplexityDecisionsConfig as PerplexityDecisionsConfig,
)
from .llms.nace.decisions.transformation import (
NaceDecisionsConfig as NaceDecisionsConfig,
)
from .llms.typesafe.decisions.transformation import (
TypeSafeDecisionsConfig as TypeSafeDecisionsConfig,
)

10
litellm/_internal_context.py
View file

@ -37,6 +37,16 @@ REDIS_FAMILIES_METADATA_KEY: Final = "families"
_service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", default=None)
@contextmanager
def internal_sub_call() -> Generator[None]:
"""A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
token: Final = is_internal_call.set(True)
try:
yield
finally:
is_internal_call.reset(token)
@contextmanager
def post_response_phase() -> Generator[None]:
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""

2
litellm/_lazy_imports_registry.py
View file

@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
"NaceDecisionsConfig",
"OpenRouterDecisionsConfig",
"FireworksAIDecisionsConfig",
"CloudflareDecisionsConfig",
@ -729,6 +730,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"HostedVLLMRerankConfig",
),
"PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"),
"NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"),
"TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"),
"OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"),
"FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"),

24
litellm/_process_defaults.py Normal file
View file

@ -0,0 +1,24 @@
import os
import warnings
import dotenv
def _dev_env_hot_reload_enabled() -> bool:
"""The proxy exports this flag when started with ``--reload``. A reloaded
worker is a fresh process that inherits the reloader's environment, so an
edited ``.env`` value stays masked by the stale inherited one unless we
let the file win; overriding makes the edit take effect on reload."""
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
def apply_process_defaults() -> None:
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
# This warning can accumulate during streaming and cause memory leaks
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())

8
litellm/assistants/main.py
View file

@ -129,6 +129,7 @@ def get_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_assistants=aget_assistants,
)
@ -312,6 +313,7 @@ def create_assistants(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
create_assistant_data=create_assistant_data,
client=client,
async_create_assistants=async_create_assistants,
@ -461,6 +463,7 @@ def delete_assistant(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
assistant_id=assistant_id,
client=client,
async_delete_assistants=async_delete_assistants,
@ -637,6 +640,7 @@ def create_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
acreate_thread=acreate_thread,
)
@ -783,6 +787,7 @@ def get_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_thread=aget_thread,
)
@ -960,6 +965,7 @@ def add_message(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
a_add_message=a_add_message,
)
@ -1111,6 +1117,7 @@ def get_messages(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
aget_messages=aget_messages,
)
@ -1308,6 +1315,7 @@ def run_thread(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=organization,
litellm_params=litellm_params_dict,
client=client,
arun_thread=arun_thread,
event_handler=event_handler,

3
litellm/batches/batch_utils.py
View file

@ -19,7 +19,7 @@ from litellm.llms.vertex_ai.batches.transformation import (
)
from litellm.types.llms.openai import Batch
from litellm.types.utils import ModelInfo, Usage
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS
from litellm.types.workload_identity import ANTHROPIC_WIF_KWARGS_KEYS, OPENAI_WIF_KWARGS_KEYS
from litellm.utils import token_counter
@ -569,6 +569,7 @@ def extract_file_access_credentials(
# A federated deployment holds no api_key, so without these the fetch that reads a
# finished batch's output has nothing to authenticate with and its cost is never billed.
*sorted(ANTHROPIC_WIF_KWARGS_KEYS),
*sorted(OPENAI_WIF_KWARGS_KEYS),
)
for key in credential_keys:
if key in litellm_params:

4
litellm/batches/main.py
View file

@ -309,6 +309,7 @@ def create_batch(
timeout=timeout,
max_retries=optional_params.max_retries,
_is_async=_is_async,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -471,6 +472,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -819,6 +821,7 @@ def list_batches(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@ -1016,6 +1019,7 @@ def cancel_batch(
organization=organization,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params,
)
elif custom_llm_provider == "azure":
api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")

13
litellm/caching/caching_handler.py
View file

@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
import litellm
from litellm._internal_context import post_response_phase
from litellm._internal_context import is_internal_call, post_response_phase
from litellm._logging import print_verbose, verbose_logger
from litellm.caching import InMemoryCache
from litellm.caching.caching import S3Cache, response_cache_phase
@ -511,8 +511,10 @@ class LLMCachingHandler:
custom_llm_provider=custom_llm_provider,
)
if not _should_defer_streaming_cache_hit_callbacks(cached_result=cached_result):
logging_obj.handle_sync_success_callbacks_for_async_calls(
if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
cached_result=cached_result
):
logging_obj.handle_sync_cache_hit_success_callbacks(
result=cached_result,
start_time=start_time,
end_time=end_time,
@ -820,7 +822,12 @@ class LLMCachingHandler:
start_time (datetime): The start time of the operation.
end_time (datetime): The end time of the operation.
cache_hit (bool): Whether it was a cache hit.
An internal sub-call's hit fires nothing, since the parent call that folds it in logs it.
"""
if is_internal_call.get():
return
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

6
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -197,7 +197,7 @@ def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool
return not _supports_audio_input(model, litellm_params)
def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
def responses_provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]:
return MappingProxyType(
{
key: value
@ -1455,7 +1455,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
model_response.id = _upstream_response_id(raw_response.id) or raw_response.id
for key, value in _provider_metadata(raw_response.model_extra).items():
for key, value in responses_provider_metadata(raw_response.model_extra).items():
setattr(model_response, key, value)
# Preserve hidden params from the ResponsesAPIResponse, especially the headers
@ -2139,7 +2139,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
from litellm.responses.utils import ResponseAPILoggingUtils
usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_data.get("usage"))
provider_metadata: Final = _provider_metadata(response_data)
provider_metadata: Final = responses_provider_metadata(response_data)
served_service_tier: Final = response_data.get("service_tier")
return ModelResponseStream(
choices=[

11
litellm/constants.py
View file

@ -1,4 +1,5 @@
import os
import re
import sys
from enum import Enum
from types import MappingProxyType
@ -18,6 +19,9 @@ MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_PROFILE: Final = "jwt_bearer_obo"
MICROSOFT_365_COPILOT_DEFAULT_TOKEN_EXCHANGE_SCOPE: Final = "https://graph.microsoft.com/.default"
# Replies depend on the caller's delegated identity, which response-cache keys do not include.
RESPONSE_CACHE_EXCLUDED_PROVIDERS: Final = frozenset({"microsoft_365_copilot", "databricks_agent"})
CLAUDE_CODE_PICKER_PATTERN: Final = re.compile(r"claude|anthropic", re.IGNORECASE)
GATEWAY_CLIENT_HEADER: Final = "x-gateway-client"
CLAUDE_CODE_CLIENT: Final = "claude-code"
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
@ -57,6 +61,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
"model_group_retry_policy",
"model_group_alias",
"enable_weighted_failover",
"enable_mid_stream_fallback_continuation",
"enable_tag_filtering",
"tag_routing_prefix",
"optional_pre_call_checks",
@ -146,6 +151,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
# Streams exceeding this duration are terminated with a Timeout error.
# None (default) = no limit. Set env var to a number of seconds to enable globally.
LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS")
MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Maximum number of base64 characters to keep in logging payloads.
# Data URIs exceeding this are replaced with a size placeholder.
@ -389,7 +395,7 @@ DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_session_update"
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY: Final = "litellm.bedrock_realtime.committed_failure"
BEDROCK_REALTIME_SDK_DISTRIBUTION: Final = "aws-sdk-bedrock-runtime"
@ -1613,6 +1619,7 @@ CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVA
MCP_TOOL_NAME_PREFIX: Final = "mcp_tool"
MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)
PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096
INTERNAL_CALLBACK_NAMES: Final = frozenset({"cache", "vector_store_pre_call_hook"})
# Headers to control callbacks
X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
@ -1629,6 +1636,7 @@ INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
"Truncation is a DB storage safeguard. "
@ -1856,6 +1864,7 @@ SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_L
PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25))
PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0))
PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0))
TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS: Final = 30
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500))

1
litellm/containers/README.md
View file

@ -120,6 +120,7 @@ class AzureContainerConfig(BaseContainerConfig):
self,
headers: dict,
api_key: Optional[str] = None,
litellm_params: Optional[GenericLiteLLMParams] = None,
) -> dict:
"""Azure uses api-key header instead of Bearer token."""
import litellm

137
litellm/cost_calculator.py
View file

@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from datetime import datetime
from functools import lru_cache
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from httpx import Response
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import (
from litellm.llms.openai.cost_calculation import (
video_output_cost_per_second,
)
from litellm.llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTAudioTranscriptionConfig,
)
from litellm.llms.perplexity.cost_calculator import (
cost_per_token as perplexity_cost_per_token,
)
@ -147,6 +150,7 @@ from litellm.utils import (
TranscriptionResponse,
cached_get_model_info_helper,
token_counter,
unmapped_model_generalization,
)
if TYPE_CHECKING:
@ -325,6 +329,15 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return False
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -863,6 +876,16 @@ def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | Non
return own_model_info["litellm_provider"]
def _pricing_provider_for_cost_candidate(
candidate_provider: str | None,
request_provider: str | None,
current_provider: object,
) -> str | None:
if candidate_provider != request_provider:
return candidate_provider
return current_provider if isinstance(current_provider, str) else None
def select_model_name_for_cost_calc(
model: str | None,
completion_response: object | None,
@ -1426,32 +1449,15 @@ def completion_cost(
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
) -> float:
"""
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
Return the USD cost of a LiteLLM call.
Parameters:
completion_response (litellm.ModelResponses): [Required] The response received from a LiteLLM completion request.
Pass `completion_response` to price a response LiteLLM returned. Without one, pass `model` plus
`messages` / `prompt` and `completion`, and token counts are computed from the text. Prices come from
`litellm.model_cost` unless `custom_cost_per_token` or `custom_cost_per_second` is given. `total_time`
is in milliseconds and only matters for per-second pricing such as Replicate and SageMaker.
[OPTIONAL PARAMS]
model (str): Optional. The name of the language model used in the completion calls
prompt (str): Optional. The input prompt passed to the llm
completion (str): Optional. The output completion text from the llm
total_time (float, int): Optional. (Only used for Replicate LLMs) The total time used for the request in seconds
custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
Returns:
float: The cost in USD dollars for the completion based on the provided parameters.
Exceptions:
Raises exception if model not in the litellm model cost map. Register model, via custom pricing or PR - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
Note:
- If completion_response is provided, the function extracts token information and the model name from it.
- If completion_response is not provided, the function calculates token counts based on the model and input text.
- The cost is calculated based on the model, prompt tokens, and completion tokens.
- For certain models containing "togethercomputer" in the name, prices are based on the model size.
- For un-mapped Replicate models, the cost is calculated based on the total time used for the request.
Raises if no candidate model name resolves to pricing. Register the model via `litellm.register_model`
or add it to model_prices_and_context_window.json.
"""
try:
call_type = infer_call_type(call_type, completion_response) or "completion"
@ -1549,14 +1555,20 @@ def completion_cost(
)
requested_model: Final = model
potential_model_names: Final = [
selected_model,
_get_response_model(completion_response),
]
if model is not None:
potential_model_names.append(model)
request_pricing_provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None
raw_candidates: Final = (
(
selected_model,
base_model_provider if base_model_provider is not None else request_pricing_provider,
),
(_get_response_model(completion_response), request_pricing_provider),
*(((model, request_pricing_provider),) if model is not None else ()),
)
potential_model_names: Final = (
raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
)
for idx, model in enumerate(potential_model_names):
for idx, (model, pricing_provider) in enumerate(potential_model_names):
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("selected model name for cost calculation: %s", model)
@ -1739,6 +1751,12 @@ def completion_cost(
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):
@ -1895,13 +1913,6 @@ def completion_cost(
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
pricing_provider: str | None = (
base_model_provider
if base_model_provider is not None and idx == 0
else custom_llm_provider
if isinstance(custom_llm_provider, str)
else None
)
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
@ -1909,7 +1920,11 @@ def completion_cost(
model=model,
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
@ -2008,7 +2023,11 @@ def completion_cost(
if cost_per_token_usage_object is not None and model:
_token_type_breakdown = get_token_type_cost_breakdown(
model=model,
custom_llm_provider=pricing_provider,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
usage=cost_per_token_usage_object,
service_tier=service_tier,
data_residency=data_residency,
@ -2048,7 +2067,7 @@ def completion_cost(
)
if idx == len(potential_model_names) - 1:
raise e
raise Exception(f"Unable to calculat cost for received potential model names - {potential_model_names}")
raise Exception(f"Unable to calculate cost for received potential model names - {potential_model_names}")
except Exception as e:
raise e
@ -2063,13 +2082,20 @@ def get_response_cost_from_hidden_params(
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
response_cost: Final = additional_headers["llm_provider-x-litellm-response-cost"]
if response_cost is None:
return None
return float(additional_headers["llm_provider-x-litellm-response-cost"])
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
def _billable_header_cost(value: object) -> float | None:
if not isinstance(value, (str, int, float)):
return None
try:
cost: Final = float(value)
except ValueError:
return None
return cost if math.isfinite(cost) and cost >= 0 else None
def response_cost_calculator(
response_object: ModelResponse
| EmbeddingResponse
@ -2263,6 +2289,25 @@ def _cost_map_model_info(model: str, custom_llm_provider: str | None) -> ModelIn
return None
_CostCandidate: TypeAlias = tuple[str | None, str | None]
def _is_unpriced_generalized_candidate(candidate: _CostCandidate) -> bool:
model, pricing_provider = candidate
if model is None:
return False
rule_info: Final = unmapped_model_generalization(model, pricing_provider)
return rule_info is not None and not _cost_map_entry_prices_anything(rule_info)
def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, ...]) -> tuple[_CostCandidate, ...]:
unpriced: Final = tuple(_is_unpriced_generalized_candidate(candidate) for candidate in candidates)
return (
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if not is_unpriced),
*(candidate for candidate, is_unpriced in zip(candidates, unpriced, strict=True) if is_unpriced),
)
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None

10
litellm/decisions/call.py
View file

@ -57,7 +57,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.utils import ProviderConfigManager
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
OpenAIDecisionQuestions: TypeAlias = Sequence[OpenAIDecisionQuestion | Mapping[str, object]]
@ -458,7 +458,7 @@ def prepare_call(
) from error
try:
resolved_api_base, resolved_api_key = provider_config.resolve_connection(
resolved_api_base, static_api_key = provider_config.resolve_connection(
dynamic_api_base or api_base, dynamic_api_key or api_key
)
except ValueError as error:
@ -469,6 +469,7 @@ def prepare_call(
model=model,
llm_provider=provider,
)
resolved_api_key: Final = provider_config.resolve_credential(static_api_key, resolved_api_base, kwargs)
if resolved_api_key is None and provider_config.api_key_required:
raise litellm.AuthenticationError(
message=f"Missing API key for Decisions provider '{provider}'",
@ -666,7 +667,10 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
{
"model": f"{call.custom_llm_provider}/{call.model}",
"custom_llm_provider": call.custom_llm_provider,
"provider_response_model": f"{call.custom_llm_provider}/{call.model}",
"provider_response_model": (
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
or f"{call.custom_llm_provider}/{call.model}"
),
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
}
)

31
litellm/decisions/main.py
View file

@ -1,5 +1,7 @@
import asyncio
import warnings
from collections.abc import Mapping
from functools import partial
from typing import Final
import httpx
@ -104,19 +106,22 @@ async def adecisions(
videos: DecisionsVideoClips | None = None,
**kwargs: object,
) -> OpenAIDecisionResponse | DecisionsResponse:
call: Final = _prepare_decisions_call(
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
call: Final = await asyncio.to_thread(
partial(
_prepare_decisions_call,
model=model,
decision_input=input,
questions=questions,
safety_identifier=safety_identifier,
audio=audio,
videos=videos,
api_key=api_key,
api_base=api_base,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
extra_headers=extra_headers,
kwargs=kwargs,
)
)
return await asend(call)

6
litellm/exceptions.py
View file

@ -313,6 +313,10 @@ class VectorStoreSearchError(BadRequestError):
)
class MalformedRequestFieldError(BadRequestError):
pass
class UnprocessableEntityError(openai.UnprocessableEntityError):
def __init__(
self,
@ -1217,6 +1221,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
num_retries: int | None = None,
generated_content: str = "",
is_pre_first_chunk: bool = False,
emitted_disqualifying_content: bool = False,
):
original_status: Final = getattr(original_exception, "status_code", None)
self.status_code = int(original_status) if original_status is not None else 503
@ -1229,6 +1234,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
self.num_retries = num_retries
self.generated_content = generated_content
self.is_pre_first_chunk = is_pre_first_chunk
self.emitted_disqualifying_content = emitted_disqualifying_content
# Create a response if one wasn't provided
if response is None:

14
litellm/experimental_mcp_client/client.py
View file

@ -498,6 +498,7 @@ class MCPClient:
elicitation_callback: "ElicitationCallback | None" = None,
logging_callback: Callable | None = None,
protocol_version: MCPUpstreamProtocol = "auto",
upstream_transport: httpx2.AsyncBaseTransport | None = None,
):
self.protocol_version: MCPUpstreamProtocol = TypeAdapter[MCPUpstreamProtocol](
MCPUpstreamProtocol
@ -524,6 +525,7 @@ class MCPClient:
self._sampling_callback: Callable | None = sampling_callback
self._elicitation_callback: ElicitationCallback | None = elicitation_callback
self._logging_callback: Callable | None = logging_callback
self._upstream_transport: httpx2.AsyncBaseTransport | None = upstream_transport
self._session_callbacks_override: _SessionCallbackMap | None = None
# handle the basic auth value if provided
if auth_value:
@ -610,7 +612,11 @@ class MCPClient:
return stdio_client(server_params), None
if self.transport_type == MCPTransport.sse:
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
return (
sse_client(
url=self.server_url,
@ -622,7 +628,11 @@ class MCPClient:
)
# HTTP transport (default)
headers = self._get_auth_headers()
httpx_client_factory = self._create_httpx_client_factory()
httpx_client_factory = (
self._create_httpx_client_factory()
if self._upstream_transport is None
else self._create_httpx_client_factory(transport=self._upstream_transport)
)
verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
http_client = httpx_client_factory(
headers=headers,

6
litellm/files/main.py
View file

@ -259,6 +259,7 @@ def create_file(
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
create_file_data=_create_file_request,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -388,6 +389,7 @@ def file_retrieve(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -569,6 +571,7 @@ def file_delete(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -775,6 +778,7 @@ def file_list(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -972,6 +976,7 @@ def file_content(
timeout=timeout,
max_retries=optional_params.max_retries,
organization=openai_creds.organization,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "azure":
azure_creds: Final = get_azure_credentials(
@ -1093,6 +1098,7 @@ def file_content_streaming(
organization=openai_creds.organization,
chunk_size=chunk_size,
client=client if isinstance(client, (OpenAI, AsyncOpenAI)) else None,
litellm_params=optional_params.model_dump(exclude_none=True),
)
elif custom_llm_provider == LlmProviders.VERTEX_AI.value:
if not _is_async:

4
litellm/fine_tuning/main.py
View file

@ -239,6 +239,7 @@ def create_fine_tuning_job(
client=kwargs.get(
"client", None
), # note, when we add this to `GenericLiteLLMParams` it impacts a lot of other tests + linting
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -436,6 +437,7 @@ def cancel_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -591,6 +593,7 @@ def list_fine_tuning_jobs(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":
@ -730,6 +733,7 @@ def retrieve_fine_tuning_job(
max_retries=optional_params.max_retries,
_is_async=_is_async,
client=kwargs.get("client", None),
litellm_params=optional_params.model_dump(exclude_none=True),
)
# Azure OpenAI
elif custom_llm_provider == "azure":

9
litellm/images/main.py
View file

@ -39,6 +39,7 @@ from openai.types.audio.transcription_create_params import FileTypes
# BFL handlers
from litellm.llms.black_forest_labs.image_edit.handler import bfl_image_edit
from litellm.llms.black_forest_labs.image_generation.handler import bfl_image_generation
from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config
from litellm.main import (
azure_chat_completions,
base_llm_aiohttp_handler,
@ -505,6 +506,7 @@ def image_generation(
aimg_generation=aimg_generation,
client=client,
headers=headers,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":
if model is None:
@ -681,10 +683,13 @@ def image_variation(
api_base = provider_config.get_api_base(litellm_params.get("api_base", None))
if image_variation_provider == LITELLM_IMAGE_VARIATION_PROVIDERS.OPENAI:
if api_key is None:
raise ValueError("API key is required for OpenAI image variations")
if api_base is None:
raise ValueError("API base is required for OpenAI image variations")
if api_key is None and (
resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
is None
):
raise ValueError("API key is required for OpenAI image variations")
response = openai_image_variations.image_variations(
model_response=model_response,

71
litellm/integrations/langfuse/langfuse.py
View file

@ -103,6 +103,43 @@ def _is_session_header_trace(trace_id: object, session_id: object, proxy_server_
)
def langfuse_trace_metadata(metadata: object, proxy_server_request: object) -> Mapping[str, object]:
entries: Final = _object_mapping(metadata) or _NO_METADATA
request: Final = _object_mapping(proxy_server_request)
headers: Final = _object_mapping(request.get("headers")) if request is not None else None
if headers is None:
return entries
header_entries: Final = {
key.removeprefix("langfuse_"): value
for key, value in _widened_items(headers)
if isinstance(key, str) and key.startswith("langfuse_")
}
return MappingProxyType({**entries, **header_entries})
def langfuse_trace_id(
metadata: Mapping[str, object],
standard_trace_id: object,
litellm_trace_id: object,
litellm_call_id: object,
proxy_server_request: object,
) -> object:
existing_trace_id: Final = metadata.get("existing_trace_id")
if existing_trace_id is not None:
return existing_trace_id
metadata_trace_id: Final = metadata.get("trace_id")
trace_id: Final = (
metadata_trace_id
if metadata_trace_id is not None
else standard_trace_id
if standard_trace_id is not None
else litellm_trace_id or litellm_call_id
)
if _is_session_header_trace(trace_id, metadata.get("session_id"), proxy_server_request):
return litellm_call_id or trace_id
return trace_id
class _UsageObject(Protocol):
"""Token-count surface the Langfuse logger reads off a response usage payload."""
@ -680,33 +717,19 @@ class LangFuseLogger:
# Add default langfuse tags
tags = self.add_default_langfuse_tags(tags=tags, kwargs=kwargs, metadata=metadata)
call_trace_id: Final = langfuse_trace_id(
metadata=clean_metadata,
standard_trace_id=standard_logging_object.get("trace_id")
if standard_logging_object is not None
else None,
litellm_trace_id=kwargs.get("litellm_trace_id"),
litellm_call_id=litellm_call_id,
proxy_server_request=litellm_params.get("proxy_server_request"),
)
session_id: Final = clean_metadata.pop("session_id", None)
trace_name = cast(str | None, clean_metadata.pop("trace_name", None))
trace_id = clean_metadata.pop("trace_id", None)
# Use standard_logging_object.trace_id if available (when trace_id from metadata is None)
# This allows standard trace_id to be used when provided in standard_logging_object
if trace_id is None and standard_logging_object is not None:
trace_id = cast(str | None, standard_logging_object.get("trace_id"))
# Fallback to litellm_call_id if no trace_id found
if trace_id is None:
trace_id = kwargs.get("litellm_trace_id") or litellm_call_id
clean_metadata.pop("trace_id", None)
existing_trace_id: Final = clean_metadata.pop("existing_trace_id", None)
# If existing_trace_id is provided, use it as the trace_id to return
# This allows continuing an existing trace while still returning the correct trace_id
if existing_trace_id is not None:
trace_id = existing_trace_id
call_trace_id: Final = (
litellm_call_id or trace_id
if existing_trace_id is None
and _is_session_header_trace(trace_id, session_id, litellm_params.get("proxy_server_request"))
else trace_id
)
if call_trace_id != trace_id:
verbose_logger.debug(
"Langfuse: trace_id %s came from a session header; using call id %s so each call gets its own trace",
trace_id,
call_trace_id,
)
requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ()))
update_trace_keys: Final = (
requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else ()

32
litellm/integrations/prometheus.py
View file

@ -27,6 +27,7 @@ from litellm.constants import (
PROXY_REJECTED_BEFORE_ROUTING_KEY,
)
from litellm.exceptions import (
MalformedRequestFieldError,
validate_rate_limit_category,
validate_rate_limit_type,
)
@ -258,6 +259,7 @@ _POSITIVE_SERIES_CAP: Final[TypeAdapter[int]] = TypeAdapter(Annotated[int, Field
_POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(gt=0)])
_SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
_DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
_ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@ -269,6 +271,15 @@ def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _Series
return None
def _gauge_reading_or_none(value: object) -> float | None:
if not isinstance(value, str | int | float):
return None
try:
return float(value)
except (ValueError, OverflowError):
return None
def _positive_or_ignored(setting: str, value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
if value is None:
return None
@ -3104,7 +3115,11 @@ class PrometheusLogger(CustomLogger):
# On LiteLLM-side rejects (no deployment picked), route request_kwargs["model"]
# into requested_model and leave deployment-scoped labels empty.
deployment_selected: Final = bool(model_id) and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
deployment_selected: Final = (
bool(model_id)
and not _litellm_params.get(PROXY_REJECTED_BEFORE_ROUTING_KEY)
and not isinstance(exception, MalformedRequestFieldError)
)
if deployment_selected:
label_litellm_model_name = litellm_model_name
label_model_id = model_id
@ -3345,12 +3360,11 @@ class PrometheusLogger(CustomLogger):
model_group=enum_values.model_group,
)
remaining_requests: int | None = None
remaining_tokens: int | None = None
if additional_headers := standard_logging_payload["hidden_params"]["additional_headers"]:
# OpenAI / OpenAI Compatible headers
remaining_requests = additional_headers.get("x_ratelimit_remaining_requests", None)
remaining_tokens = additional_headers.get("x_ratelimit_remaining_tokens", None)
additional_headers: Final = _ADDITIONAL_HEADERS.validate_python(
standard_logging_payload["hidden_params"]["additional_headers"] or {}
)
remaining_requests: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_requests"))
remaining_tokens: Final = _gauge_reading_or_none(additional_headers.get("x_ratelimit_remaining_tokens"))
if litellm_overhead_time_ms := standard_logging_payload["hidden_params"].get("litellm_overhead_time_ms"):
_labels = prometheus_label_factory(
@ -3368,7 +3382,7 @@ class PrometheusLogger(CustomLogger):
label_context=label_context,
)
if remaining_requests:
if remaining_requests is not None:
"""
"model_group",
"api_provider",
@ -3382,7 +3396,7 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_requests_metric.labels(**_labels).set(remaining_requests)
if remaining_tokens:
if remaining_tokens is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_remaining_tokens_metric"),
enum_values=enum_values,

2
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -44,7 +44,7 @@ from litellm.litellm_core_utils.private_json import (
stage_private_json,
write_private_json,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.base_model import LiteLLMBaseModel
@dataclass(frozen=True, slots=True)

4
litellm/litellm_core_utils/core_helpers.py
View file

@ -318,7 +318,7 @@ def remove_items_at_indices(items: _PoppableByIndex | None, indices: Iterable[in
items.pop(index)
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: dict) -> dict:
def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metadata: Mapping[str, object]) -> dict:
"""
Helper to get litellm metadata for spend tracking
@ -326,7 +326,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad
and user_api_key values are in 'metadata'.
"""
potential_spend_tracking_metadata_substring: Final = "user_api_key"
for key, value in metadata.items():
for key, value in dict(metadata).items():
if potential_spend_tracking_metadata_substring in key:
litellm_metadata[key] = value
return litellm_metadata

1
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -78,6 +78,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_auth_type",
"github_copilot_user_session",
"fireworks_forward_user_id",
"responses_api_version",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
}
)

79
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -17,6 +17,7 @@ import random
import sys
import threading
import time
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, replace
from datetime import datetime, timezone
@ -24,11 +25,11 @@ from importlib.resources import files
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, Protocol
from typing import Final, Protocol, cast # noqa: TID251 # adapts boto3 and botocore's dynamic client responses
import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm import verbose_logger
from litellm.constants import (
@ -266,6 +267,19 @@ class _SyncGetClient(Protocol):
def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: ...
class _S3Body(Protocol):
def read(self) -> bytes: ...
class _S3GetObjectResponse(TypedDict):
Body: ReadOnly[_S3Body]
ETag: NotRequired[ReadOnly[str]]
class _S3Client(Protocol):
def get_object(self, *, Bucket: str, Key: str) -> _S3GetObjectResponse: ...
_FetchAttemptOutcome = ModelCostMapReloaded | ModelCostMapReloadUnavailable | _FetchAttemptRetryable
@ -276,6 +290,20 @@ def _default_reload_client() -> _AsyncGetClient:
return get_async_httpx_client(llm_provider=httpxSpecialProvider.ModelCostMap)
def _default_s3_client(timeout: int) -> _S3Client:
import boto3
from botocore.config import Config
return cast( # cast-ok: boto3's generic client factory does not expose its generated S3 response type
_S3Client,
boto3.client( # pyright: ignore[reportUnknownMemberType] # generic factory returns Unknown service clients
"s3",
region_name=os.getenv("AWS_REGION_NAME") or None,
config=Config(connect_timeout=timeout, read_timeout=timeout, retries={"total_max_attempts": 1}),
),
)
def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -> _FetchAttemptOutcome:
reason: Final = f"{type(error).__name__} fetching {url}: {error}"
if isinstance(error, (httpx.InvalidURL, httpx.UnsupportedProtocol)):
@ -283,7 +311,41 @@ def _classify_fetch_error(error: httpx.HTTPError | httpx.InvalidURL, url: str) -
return _FetchAttemptRetryable(reason=reason, retry_after_seconds=None)
def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _FetchAttemptOutcome:
try:
parsed: Final[object] = json.loads(content)
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
model_cost_map: Final = cast( # cast-ok: JSON object keys are strings by definition
dict[str, object], parsed
)
return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
def _attempt_fetch_s3(url: str, timeout: int) -> _FetchAttemptOutcome:
from botocore.exceptions import BotoCoreError, ClientError
parts: Final = urllib.parse.urlsplit(url)
bucket: Final = parts.netloc
key: Final = parts.path.lstrip("/")
if not bucket or not key:
return ModelCostMapReloadUnavailable(reason=f"invalid S3 URL {url}: expected s3://bucket/key")
try:
client: Final = _default_s3_client(timeout)
response: Final = client.get_object(Bucket=bucket, Key=key)
content: Final = response["Body"].read()
except (BotoCoreError, ClientError) as error:
return ModelCostMapReloadUnavailable(reason=f"S3 {type(error).__name__} fetching {url}: {error}")
return _classify_fetch_content(content, url, response.get("ETag"))
async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
if parts.scheme == "s3":
return await asyncio.to_thread(_attempt_fetch_s3, url, timeout)
try:
response: Final = await client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -292,6 +354,9 @@ async def _attempt_fetch(client: _AsyncGetClient, url: str, timeout: int) -> _Fe
def _attempt_fetch_sync(client: _SyncGetClient, url: str, timeout: int) -> _FetchAttemptOutcome:
parts: Final = urllib.parse.urlsplit(url)
if parts.scheme == "s3":
return _attempt_fetch_s3(url, timeout)
try:
response: Final = client.get(url, timeout=timeout)
except (httpx.HTTPError, httpx.InvalidURL) as e:
@ -307,15 +372,7 @@ def _classify_fetch_response(response: httpx.Response, url: str) -> _FetchAttemp
)
if response.is_error:
return ModelCostMapReloadUnavailable(reason=f"HTTP {response.status_code} from {url}")
try:
parsed: Final = response.json()
except ValueError as e:
return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
if not isinstance(parsed, dict):
return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
return ModelCostMapReloaded(
model_cost_map=parsed, revision=git_blob_id(response.content), etag=response.headers.get("etag")
)
return _classify_fetch_content(response.content, url, response.headers.get("etag"))
def _next_retry_wait(

5
litellm/litellm_core_utils/hidden_params.py
View file

@ -77,6 +77,11 @@ def get_hidden_params(obj: object) -> MutableMapping[str, object] | None:
return _as_hidden_params_mapping(hidden_params)
def served_from_cache(obj: object) -> bool:
hidden_params: Final = get_hidden_params(obj)
return hidden_params is not None and hidden_params.get("cache_hit") is True
def set_hidden_params(obj: object, hidden_params: dict[str, object] | HiddenParams) -> None:
if isinstance(obj, dict):
obj[_HIDDEN_PARAMS_ATTR] = hidden_params

396
litellm/litellm_core_utils/litellm_logging.py
View file

@ -41,6 +41,7 @@ from litellm.constants import (
DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
EMPTY_MAPPING,
INTERNAL_CALLBACK_NAMES,
PROVIDER_REQUEST_ID_HEADERS,
REDACTED_BY_LITELLM,
)
@ -79,7 +80,12 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_param
from litellm.litellm_core_utils.hidden_params import (
HIDDEN_PARAMS_ATTR,
get_hidden_params,
served_from_cache,
set_hidden_param,
)
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
is_unbilled_non_inference_call,
@ -2056,7 +2062,7 @@ class Logging(LiteLLMLoggingBaseClass):
) -> None:
if response_cost is None and not calculation_failed:
return
if self.model_call_details.get("cache_hit") is True:
if self.model_call_details.get("cache_hit") is True or _served_from_cache_at_no_charge(result):
self.model_call_details["zero_cost_diagnostic"] = None
return
try:
@ -2287,6 +2293,7 @@ class Logging(LiteLLMLoggingBaseClass):
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
for_async_request=True,
**kwargs,
)
@ -2295,6 +2302,54 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["combined_usage_object"] = usage
self.model_call_details["response_cost"] = response_cost
def attempt_scoped_copy(self) -> "Logging":
"""A copy with its own model_call_details, so logging one attempt of a request through it leaves
the state the request's other attempts share untouched."""
attempt: Final = copy.copy(self)
attempt.model_call_details = dict(self.model_call_details)
return attempt
def carry_failed_attempt_usage(self) -> None:
"""Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its
own usage while the request's failure log still bills every attempt."""
usage: Final = self.model_call_details.pop("combined_usage_object", None)
response_cost: Final = self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
)
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
self.model_call_details["continued_partial_usage"] = usage
self.model_call_details["continued_partial_cost"] = response_cost
self.model_call_details["continued_response_id"] = response_id
def _add_continued_partial_spend(self, response: object) -> None:
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
own_usage: Final = getattr(response, "usage", None)
own_cost: Final = self.model_call_details.get("response_cost")
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
return
from litellm.cost_calculator import BaseTokenUsageProcessor
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, partial_usage]
)
if not isinstance(own_cost, (int, float)):
return
self.model_call_details["response_cost"] = own_cost + partial_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
total_cost=own_cost + partial_cost,
additional_costs={
**self.cost_breakdown.get("additional_costs", {}),
"mid_stream_partial_cost": partial_cost,
},
)
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
usage: Final = getattr(assembled, "usage", None)
@ -2480,7 +2535,8 @@ class Logging(LiteLLMLoggingBaseClass):
start_time,
end_time,
build_logging_payload: bool = True,
):
publish_logging_payload: bool = True,
) -> StandardLoggingPayload | None:
"""Resolve hidden params, compute response cost, and emit the standard logging payload."""
self._surface_response_headers_from_result(logging_result)
hidden_params: Final = getattr(logging_result, HIDDEN_PARAMS_ATTR, {})
@ -2507,17 +2563,17 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
if not build_logging_payload:
return
return None
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logging_result, start_time, end_time
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
standard_logging_payload: Final = self._build_standard_logging_payload(
logging_result, start_time, end_time, isolate_call_details=not publish_logging_payload
)
if not publish_logging_payload:
return standard_logging_payload
self.model_call_details["standard_logging_object"] = standard_logging_payload
if standard_logging_payload is not None:
emit_standard_logging_payload(standard_logging_payload)
return standard_logging_payload
async def _prepare_baseline_cache_estimate(self, response_obj: object) -> None:
if self.baseline_cache_context is None:
@ -2533,12 +2589,16 @@ class Logging(LiteLLMLoggingBaseClass):
await invalidate_baseline_cache(self, reason, completed=completed)
def _build_standard_logging_payload(
self, init_response_obj: object, start_time: dt_object, end_time: dt_object
self,
init_response_obj: object,
start_time: dt_object,
end_time: dt_object,
isolate_call_details: bool = False,
) -> StandardLoggingPayload | None:
"""Build StandardLoggingPayload and accumulate its construction time."""
_start: Final = time.time()
payload: Final = get_standard_logging_object_payload(
kwargs=self.model_call_details,
kwargs=dict(self.model_call_details) if isolate_call_details else self.model_call_details,
init_response_obj=init_response_obj,
start_time=start_time,
end_time=end_time,
@ -2555,11 +2615,16 @@ class Logging(LiteLLMLoggingBaseClass):
emit_standard_logging_payload(stored_payload)
def _transform_usage_objects(self, result: object) -> object:
return self._transform_usage_objects_for_payload(result, self.model_call_details.get("standard_logging_object"))
def _transform_usage_objects_for_payload(
self, result: object, standard_logging_payload: StandardLoggingPayload | None
) -> object:
if isinstance(result, ResponsesAPIResponse):
result = result.model_copy()
transformed_usage = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(result.usage)
setattr(result, "usage", transformed_usage)
if (standard_logging_payload := self.model_call_details.get("standard_logging_object")) is not None:
if standard_logging_payload is not None:
response_dict: Final = result.model_dump() if hasattr(result, "model_dump") else dict(result)
# Ensure usage is properly included with transformed chat format
if transformed_usage is not None:
@ -2568,7 +2633,7 @@ class Logging(LiteLLMLoggingBaseClass):
if hasattr(transformed_usage, "model_dump")
else dict(transformed_usage)
)
standard_logging_payload["response"] = response_dict
standard_logging_payload["response"] = response_dict # rebind-ok: mirrors the transformed usage
elif isinstance(result, TranscriptionResponse):
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
@ -2590,6 +2655,15 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"model": model})
def _with_continued_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
response_id: Final = self.model_call_details.get("continued_response_id")
if not isinstance(response_id, str) or getattr(response, "id", None) == response_id:
return response
return response.model_copy(update={"id": response_id})
def _success_handler_helper_fn(
self,
result: object = None,
@ -2599,6 +2673,27 @@ class Logging(LiteLLMLoggingBaseClass):
standard_logging_object: StandardLoggingPayload | None = None,
build_logging_payload: bool = True,
) -> tuple[datetime.datetime, datetime.datetime, object]:
prepared: Final = self._prepare_success_logging(
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
standard_logging_object=standard_logging_object,
build_logging_payload=build_logging_payload,
publish_logging_payload=True,
)
return prepared[0], prepared[1], prepared[2]
def _prepare_success_logging(
self,
result: object,
start_time: datetime.datetime | None,
end_time: datetime.datetime | None,
cache_hit: bool | None,
standard_logging_object: StandardLoggingPayload | None,
build_logging_payload: bool,
publish_logging_payload: bool,
) -> tuple[datetime.datetime, datetime.datetime, object, StandardLoggingPayload | None]:
try:
if start_time is None:
start_time = self.start_time
@ -2623,35 +2718,29 @@ class Logging(LiteLLMLoggingBaseClass):
result = self._handle_a2a_response_logging(result=result)
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
self.model_call_details["cache_hit"] = True
if isinstance(result, Response) and isinstance(
logging_result, (ModelResponse, EmbeddingResponse, TextCompletionResponse)
):
result = logging_result
if standard_logging_object is None and result is not None and self.stream is not True:
if self._is_recognized_call_type_for_logging(logging_result=logging_result) or isinstance(
logging_result, (dict, list)
):
self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
build_logging_payload=build_logging_payload,
)
elif standard_logging_object is not None:
self.model_call_details["standard_logging_object"] = standard_logging_object
else:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
# established one. A stream that assembles into a response
# object recomputes the cost right after this; a pass-through
# stream cannot (its body is opaque) and carries the cost its
# upstream reported in the response headers, which an
# unconditional reset would discard.
self.model_call_details.setdefault("response_cost", None)
standard_logging_payload: Final = self._resolve_success_logging_payload(
result=result,
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
standard_logging_object=standard_logging_object,
build_logging_payload=build_logging_payload,
publish_logging_payload=publish_logging_payload,
)
result = self._transform_usage_objects(result=result)
result = (
self._transform_usage_objects(result=result)
if publish_logging_payload
else self._transform_usage_objects_for_payload(result, standard_logging_payload)
)
if (
litellm.max_budget
@ -2670,10 +2759,47 @@ class Logging(LiteLLMLoggingBaseClass):
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
return start_time, end_time, result
return start_time, end_time, result, standard_logging_payload
except Exception as e:
raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {e}")
def _resolve_success_logging_payload(
self,
result: object,
logging_result: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
standard_logging_object: StandardLoggingPayload | None,
build_logging_payload: bool,
publish_logging_payload: bool,
) -> StandardLoggingPayload | None:
if standard_logging_object is not None:
if not publish_logging_payload:
return standard_logging_object.copy()
self.model_call_details["standard_logging_object"] = standard_logging_object
return standard_logging_object
if result is None or self.stream is True:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
# established one. A stream that assembles into a response
# object recomputes the cost right after this; a pass-through
# stream cannot (its body is opaque) and carries the cost its
# upstream reported in the response headers, which an
# unconditional reset would discard.
self.model_call_details.setdefault("response_cost", None)
return None
if not self._is_recognized_call_type_for_logging(logging_result=logging_result) and not isinstance(
logging_result, (dict, list)
):
return None
return self._process_hidden_params_and_response_cost(
logging_result=logging_result,
start_time=start_time,
end_time=end_time,
build_logging_payload=build_logging_payload,
publish_logging_payload=publish_logging_payload,
)
def _is_recognized_call_type_for_logging(
self,
logging_result: object,
@ -2807,11 +2933,14 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
self.success_handler(result=complete_streaming_response)
async def async_flush_passthrough_collected_chunks(self, collector: "PassthroughStreamCollector"):
async def async_flush_passthrough_collected_chunks(self, collector: "PassthroughStreamCollector") -> bool:
"""Log the response the collector built from the stream, and say whether there was one to log."""
complete_streaming_response: Final = collector.build_logged_response(litellm_logging_obj=self)
if complete_streaming_response is not None:
await self.async_success_handler(result=complete_streaming_response)
if complete_streaming_response is None:
return False
await self.async_success_handler(result=complete_streaming_response)
return True
def _restore_correlation_context(self) -> None:
"""Restore trace_id/session_id contextvars to their pre-call value.
@ -2855,6 +2984,8 @@ class Logging(LiteLLMLoggingBaseClass):
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
cache_hit: bool | None = None,
*,
for_async_request: bool = False,
**kwargs: object, # kwargs-ok: forwarded to _success_handler_body
) -> None:
"""Restores trace_id/session_id contextvars once this attempt's own success
@ -2862,7 +2993,12 @@ class Logging(LiteLLMLoggingBaseClass):
try:
with post_response_phase():
return self._success_handler_body(
result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit, **kwargs
result=result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
for_async_request=for_async_request,
**kwargs,
)
finally:
self._restore_correlation_context()
@ -2873,17 +3009,20 @@ class Logging(LiteLLMLoggingBaseClass):
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
cache_hit: bool | None = None,
for_async_request: bool = False,
**kwargs: Any, # kwargs-ok: forwarded from success_handler
) -> None:
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
start_time, end_time, result = self._success_handler_helper_fn(
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
start_time=start_time,
end_time=end_time,
result=result,
cache_hit=cache_hit,
standard_logging_object=kwargs.get("standard_logging_object", None),
build_logging_payload=True,
publish_logging_payload=not for_async_request,
)
litellm_params = self.model_call_details.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
@ -2908,20 +3047,52 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
if not for_async_request:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
logged_streaming_response, start_time, end_time
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
)
if standard_logging_payload is not None:
# Only emit for sync requests (async_success_handler handles async)
if is_sync_request:
emit_standard_logging_payload(standard_logging_payload)
callback_logging_payload: Final = (
self._build_standard_logging_payload(
self.model_call_details["complete_streaming_response"],
start_time,
end_time,
isolate_call_details=True,
)
standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
if for_async_request and complete_streaming_response is not None
else request_logging_payload
)
shared_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get(
"standard_logging_object"
)
isolated_logging_payload: Final = (
callback_logging_payload
if callback_logging_payload is not None or shared_logging_payload is None
else shared_logging_payload.copy()
)
call_details: dict[str, Any] # mutable-ok: legacy loggers take a plain dict
call_details = ( # rebind-ok: logging hooks may return a new dict
self.model_call_details
if not for_async_request
else (
dict(self.model_call_details)
if isolated_logging_payload is None
else {**self.model_call_details, "standard_logging_object": isolated_logging_payload}
)
if standard_logging_payload is not None:
# Only emit for sync requests (async_success_handler handles async)
if is_sync_request:
emit_standard_logging_payload(standard_logging_payload)
)
callbacks: Final = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
global_callbacks=litellm.success_callback,
@ -2929,7 +3100,7 @@ class Logging(LiteLLMLoggingBaseClass):
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
model_call_details=call_details,
result=result,
)
## LOGGING HOOK ##
@ -2939,7 +3110,7 @@ class Logging(LiteLLMLoggingBaseClass):
if (
callback.should_run_guardrail(
data=self.model_call_details,
data=call_details,
event_type=GuardrailEventHooks.logging_only,
)
is not True
@ -2947,18 +3118,21 @@ class Logging(LiteLLMLoggingBaseClass):
continue
self.shadow_eval_request_snapshot = None
self.model_call_details, result = callback.logging_hook(
kwargs=self.model_call_details,
call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict
kwargs=call_details,
result=result,
call_type=self.call_type,
)
elif isinstance(callback, CustomLogger):
self.model_call_details, result = callback.logging_hook(
kwargs=self.model_call_details,
call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict
kwargs=call_details,
result=result,
call_type=self.call_type,
)
if not for_async_request:
self.model_call_details = call_details
call_details_view: Final = MappingProxyType[str, object](call_details)
self.mark_logging_complete(event_type="sync_success")
for callback in callbacks:
try:
@ -2972,7 +3146,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "promptlayer" and promptLayerLogger is not None:
print_verbose("reaches promptlayer for logging!")
promptLayerLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -2980,7 +3154,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "supabase" and supabaseClient is not None:
print_verbose("reaches supabase for logging!")
kwargs = self.model_call_details
kwargs = call_details
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
if self.stream:
@ -2988,10 +3162,10 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches supabase for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
model = self._model_call_details_view["model"]
messages = self._model_call_details_view["messages"]
model = call_details_view["model"]
messages = call_details_view["messages"]
optional_params = kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {})
supabaseClient.log_event(
@ -3011,7 +3185,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "wandb" and weightsBiasesLogger is not None:
print_verbose("reaches wandb for logging!")
weightsBiasesLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3020,7 +3194,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "logfire" and logfireLogger is not None:
verbose_logger.debug("reaches logfire for success logging!")
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
@ -3030,10 +3204,10 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches logfire for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
logfireLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3044,7 +3218,7 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "lunary" and lunaryLogger is not None:
print_verbose("reaches lunary for logging!")
model = self.model
kwargs = self.model_call_details
kwargs = call_details
input = kwargs.get("messages", kwargs.get("input", None))
@ -3055,7 +3229,7 @@ class Logging(LiteLLMLoggingBaseClass):
if "complete_streaming_response" not in kwargs:
continue
else:
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
lunaryLogger.log_event(
type=type,
@ -3065,7 +3239,7 @@ class Logging(LiteLLMLoggingBaseClass):
input=input,
user_id=kwargs.get("user", None),
# user_props=self.model_call_details.get("user_props", None),
extra=self._model_call_details_view.get("optional_params", {}),
extra=call_details_view.get("optional_params", {}),
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3075,8 +3249,8 @@ class Logging(LiteLLMLoggingBaseClass):
if callback == "helicone" and heliconeLogger is not None:
print_verbose("reaches helicone for logging!")
model = self.model
messages = self._model_call_details_view["input"]
kwargs = self.model_call_details
messages = call_details_view["input"]
kwargs = call_details
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
if self.stream:
@ -3084,7 +3258,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches helicone for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
heliconeLogger.log_success(
model=model,
@ -3099,7 +3273,7 @@ class Logging(LiteLLMLoggingBaseClass):
global langFuseLogger
print_verbose("reaches langfuse for success logging!")
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
@ -3112,7 +3286,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches langfuse for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
langfuse_logger_to_use = LangFuseHandler.get_langfuse_logger_for_request(
globalLangfuseLogger=langFuseLogger,
@ -3137,7 +3311,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "greenscale" and greenscaleLogger is not None:
kwargs = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response": # copy.deepcopy raises errors as this could be a coroutine
kwargs[k] = v
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
@ -3150,7 +3324,7 @@ class Logging(LiteLLMLoggingBaseClass):
continue
else:
print_verbose("reaches greenscale for streaming logging!")
result = self._model_call_details_view["complete_streaming_response"]
result = call_details_view["complete_streaming_response"]
greenscaleLogger.log_event(
kwargs=kwargs,
@ -3161,7 +3335,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "athina" and athinaLogger is not None:
deep_copy = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
deep_copy[k] = v
athinaLogger.log_event(
kwargs=deep_copy,
@ -3172,7 +3346,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
if callback == "traceloop":
deep_copy = {}
for k, v in self._model_call_details_view.items():
for k, v in call_details_view.items():
if k != "original_response":
deep_copy[k] = v
traceloopLogger.log_event(
@ -3188,11 +3362,11 @@ class Logging(LiteLLMLoggingBaseClass):
if s3Logger is None:
s3Logger = S3Logger()
if self.stream:
if "complete_streaming_response" in self.model_call_details:
if "complete_streaming_response" in call_details:
print_verbose("S3Logger Logger: Got Stream Event - Completed Stream Response")
s3Logger.log_event(
kwargs=self.model_call_details,
response_obj=self.model_call_details["complete_streaming_response"],
kwargs=call_details,
response_obj=call_details["complete_streaming_response"],
start_time=start_time,
end_time=end_time,
print_verbose=print_verbose,
@ -3201,7 +3375,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose("S3Logger Logger: Got Stream Event - No complete stream response as yet")
else:
s3Logger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3215,19 +3389,17 @@ class Logging(LiteLLMLoggingBaseClass):
openMeterLogger = OpenMeterLogger()
if self.stream and complete_streaming_response is None:
openMeterLogger.log_stream_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self._model_call_details_view["complete_response"]
call_details["complete_response"] = call_details.get("complete_streaming_response", {})
result = call_details_view["complete_response"]
openMeterLogger.log_success_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3240,20 +3412,18 @@ class Logging(LiteLLMLoggingBaseClass):
): # custom logger class
if self.stream and complete_streaming_response is None:
callback.log_stream_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self._model_call_details_view["complete_response"]
call_details["complete_response"] = call_details.get("complete_streaming_response", {})
result = call_details_view["complete_response"]
callback.log_success_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3264,7 +3434,7 @@ class Logging(LiteLLMLoggingBaseClass):
print_verbose(f"success callbacks: Running Custom Callback Function - {callback}")
customLogger.log_event(
kwargs=self.model_call_details,
kwargs=call_details,
response_obj=result,
start_time=start_time,
end_time=end_time,
@ -3445,12 +3615,15 @@ class Logging(LiteLLMLoggingBaseClass):
self.model,
)
self.model_call_details["response_cost"] = None
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
@ -4102,8 +4275,18 @@ class Logging(LiteLLMLoggingBaseClass):
start_time,
end_time,
cache_hit,
for_async_request=True,
)
def handle_sync_cache_hit_success_callbacks(
self,
result: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
cache_hit: bool | None = None,
) -> None:
executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
def _should_run_sync_callbacks_for_async_calls(self) -> bool:
"""
Returns:
@ -4115,7 +4298,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
_filtered_success_callbacks = self._remove_internal_custom_logger_callbacks(_combined_sync_callbacks)
_filtered_success_callbacks = self._remove_internal_litellm_callbacks(_filtered_success_callbacks)
return len(_filtered_success_callbacks) > 0
return any(
not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES)
for callback in _filtered_success_callbacks # pyright: ignore[reportUnknownVariableType] # callback lists are untyped
)
def _should_run_sync_failure_callbacks_for_async_calls(self) -> bool:
"""
@ -4282,6 +4468,9 @@ class Logging(LiteLLMLoggingBaseClass):
row is built from this response rather than re-parsed from the served body."""
self.bridged_model_response = response
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
def billable_result(self, result: object) -> object:
"""The response the cost calculator prices: a /v1/messages body served through the chat
bridge is replaced by the ModelResponse its spend row is built from, anything else is
@ -4382,6 +4571,7 @@ class Logging(LiteLLMLoggingBaseClass):
"""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
responses_provider_metadata,
)
served_id: Final = _provider_response_id(result)
@ -4406,7 +4596,9 @@ class Logging(LiteLLMLoggingBaseClass):
"usage-only ModelResponse to keep the spend_logs row.",
str(e),
)
model_response: Final = litellm.ModelResponse(id=served_id)
model_response: Final = litellm.ModelResponse(
id=served_id, **responses_provider_metadata(result.model_extra)
)
model_response.model = self.model
usage: Final = result.usage
if usage is not None and ResponseAPILoggingUtils.is_response_api_usage(usage):
@ -6634,6 +6826,11 @@ def _get_status_fields(
return StandardLoggingPayloadStatusFields(llm_api_status=llm_api_status, guardrail_status=guardrail_status)
def _served_from_cache_at_no_charge(result: object) -> bool:
hidden_params: Final = get_hidden_params(result)
return served_from_cache(result) and hidden_params is not None and hidden_params.get("response_cost") == 0
def _extract_response_obj_and_hidden_params(
init_response_obj: object,
original_exception: Exception | None,
@ -6796,8 +6993,9 @@ def get_standard_logging_object_payload(
) # maintain backwards compatibility with old request body check
saved_cache_cost: float = 0.0
if cache_hit is True:
if cache_hit is True or served_from_cache(init_response_obj):
id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id
if cache_hit is True:
saved_cache_cost = (
logging_obj.response_cost_calculator(
result=init_response_obj,

13
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -858,9 +858,9 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def get_web_search_options(kwargs: Mapping[str, object]) -> WebSearchOptions | None:
if "web_search_options" in kwargs:
web_search_options: Final = cast(WebSearchOptions, kwargs.get("web_search_options", {}))
return WebSearchOptions(**web_search_options)
web_search_options: Final = kwargs.get("web_search_options")
if isinstance(web_search_options, dict):
return WebSearchOptions(**cast(WebSearchOptions, web_search_options))
tools: Final = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
kwargs=kwargs, tool_type="web_search_preview"
@ -876,10 +876,9 @@ class StandardBuiltInToolCostTracking:
_get_web_search_options = get_web_search_options
@staticmethod
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> list[object] | None:
if "tools" in kwargs:
return cast(list[object], kwargs.get("tools", []))
return None
def _get_tools_from_kwargs(kwargs: Mapping[str, object], tool_type: str) -> Sequence[object] | None:
tools: Final = kwargs.get("tools")
return cast(Sequence[object], tools) if isinstance(tools, (list, tuple)) else None
@staticmethod
def get_file_search_tool_call(kwargs: Mapping[str, object]) -> FileSearchTool | None:

72
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -251,14 +251,7 @@ async def convert_to_streaming_response_async(
setattr(
model_response_object,
"usage",
Usage(
completion_tokens=response_object["usage"].get("completion_tokens", 0),
prompt_tokens=response_object["usage"].get("prompt_tokens", 0),
total_tokens=response_object["usage"].get("total_tokens", 0),
completion_tokens_details=_cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
),
),
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -345,12 +338,10 @@ def convert_to_streaming_response(
model_response_object.choices = choice_list
if "usage" in response_object and response_object["usage"] is not None:
setattr(model_response_object, "usage", Usage())
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0)
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0)
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0)
model_response_object.usage.completion_tokens_details = _cached_completion_tokens_details(
_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])
setattr(
model_response_object,
"usage",
_replayed_usage(_CACHED_USAGE_DETAILS.validate_python(response_object["usage"])),
)
if "id" in response_object:
@ -414,13 +405,28 @@ class _CachedMessageProviderFields(TypedDict, total=False):
class _CachedUsageDetails(TypedDict, total=False):
completion_tokens: ReadOnly[object]
prompt_tokens: ReadOnly[object]
total_tokens: ReadOnly[object]
completion_tokens_details: ReadOnly[object]
prompt_tokens_details: ReadOnly[object]
cache_read_input_tokens: ReadOnly[object]
cache_creation_input_tokens: ReadOnly[object]
class _PromptCacheTokenCounts(TypedDict, total=False):
cache_read_input_tokens: ReadOnly[int]
cache_creation_input_tokens: ReadOnly[int]
_CACHED_MESSAGE_PROVIDER_FIELDS: Final = TypeAdapter(_CachedMessageProviderFields)
_CACHED_USAGE_DETAILS: Final = TypeAdapter(_CachedUsageDetails)
_FIELD_MAP: Final = TypeAdapter(dict[str, object])
_COMPLETION_TOKENS_DETAILS: Final = TypeAdapter(CompletionTokensDetailsWrapper)
_PROMPT_TOKENS_DETAILS: Final = TypeAdapter(PromptTokensDetailsWrapper)
_PROMPT_CACHE_TOKEN_COUNTS: Final = TypeAdapter(_PromptCacheTokenCounts)
_TOKEN_COUNT: Final = TypeAdapter(int)
_PROMPT_CACHE_TOKEN_FIELDS: Final = ("cache_read_input_tokens", "cache_creation_input_tokens")
def _cached_provider_specific_fields(message: _CachedMessageProviderFields) -> Mapping[str, object] | None:
@ -438,6 +444,44 @@ def _cached_completion_tokens_details(usage: _CachedUsageDetails) -> CompletionT
return None
def _cached_prompt_tokens_details(usage: _CachedUsageDetails) -> PromptTokensDetailsWrapper | None:
details: Final = usage.get("prompt_tokens_details")
if not isinstance(details, dict):
return None
try:
return _PROMPT_TOKENS_DETAILS.validate_python(details)
except ValidationError:
return None
def _cached_token_count(count: object) -> int | None:
try:
return _TOKEN_COUNT.validate_python(count)
except ValidationError:
return None
def _cached_prompt_cache_token_counts(usage: _CachedUsageDetails) -> _PromptCacheTokenCounts:
return _PROMPT_CACHE_TOKEN_COUNTS.validate_python(
{
field: count
for field in _PROMPT_CACHE_TOKEN_FIELDS
if (count := _cached_token_count(usage.get(field))) is not None
}
)
def _replayed_usage(usage: _CachedUsageDetails) -> Usage:
return Usage(
completion_tokens=_cached_token_count(usage.get("completion_tokens")) or 0,
prompt_tokens=_cached_token_count(usage.get("prompt_tokens")) or 0,
total_tokens=_cached_token_count(usage.get("total_tokens")) or 0,
completion_tokens_details=_cached_completion_tokens_details(usage),
prompt_tokens_details=_cached_prompt_tokens_details(usage),
**_cached_prompt_cache_token_counts(usage),
)
def handle_invalid_parallel_tool_calls(
tool_calls: list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall],
) -> list[ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall] | None:

125
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -9,7 +9,8 @@ import json
import mimetypes
import re
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from itertools import groupby, islice
from graphlib import CycleError, TopologicalSorter
from itertools import chain, groupby, islice
from os import PathLike
from pathlib import Path
from types import MappingProxyType
@ -1426,6 +1427,123 @@ def _schema_levels(schema: Mapping[str, object]) -> Iterator[tuple[Mapping[str,
frontier = tuple(child for node in frontier for child in _subschemas(node))
def schema_fragment_ref(ref: str) -> str:
return f"#{ref}" if ref.startswith(("/$defs/", "/definitions/")) else ref
def schema_has_unresolvable_ref(schema: Mapping[str, object]) -> bool:
definitions: Final = MappingProxyType(dict(_root_definitions(schema)))
if not _refs_in(schema) <= definitions.keys():
return True
try:
tuple(TopologicalSorter({pointer: _refs_in(body) for pointer, body in definitions.items()}).static_order())
except CycleError:
return True
return False
def schema_union_property_count(schema: Mapping[str, object]) -> int:
return sum(1 for _, node in _property_schemas(schema) if _is_union(node))
def schema_optional_property_count(schema: Mapping[str, object]) -> int:
return sum(len(_optional_property_names(node)) for node in _schema_nodes(schema))
def schema_property_count(schema: Mapping[str, object]) -> int:
return sum(len(properties) for node in _schema_nodes(schema) if (properties := _schema_at(node, "properties")))
def schema_grammar_branch_count(schema: Mapping[str, object]) -> int:
return sum(_grammar_branches(node) for node in _schema_nodes(schema))
def schema_has_open_object(schema: Mapping[str, object]) -> bool:
return any(_is_open_object(node) for node in _schema_nodes(schema))
def schema_has_lookaround_pattern(schema: Mapping[str, object]) -> bool:
return any(
isinstance(pattern := node.get("pattern"), str) and _uses_regex_lookaround(pattern)
for node in _schema_nodes(schema)
)
def schema_string_formats(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
value for node in _schema_nodes(schema) if _may_be_string(node) and isinstance(value := node.get("format"), str)
)
def _may_be_string(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type is None or node_type == "string" or "string" in _list_at(node, "type")
def is_object_schema(node: Mapping[str, object]) -> bool:
node_type: Final = node.get("type")
return node_type == "object" or (isinstance(node_type, list) and "object" in node_type)
def _schema_nodes(schema: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
return chain.from_iterable(islice(_schema_levels(schema), _MAX_SCHEMA_NESTING))
_SCHEMA_LIST: Final = TypeAdapter(tuple[object, ...])
def _list_at(container: Mapping[str, object], key: str) -> tuple[object, ...]:
value: Final = container.get(key)
return _SCHEMA_LIST.validate_python(value) if isinstance(value, list) else ()
def _named_schemas(container: Mapping[str, object] | None) -> Iterator[tuple[str, Mapping[str, object]]]:
if container is None:
return iter(())
return ((key, value) for key in container if (value := _schema_at(container, key)) is not None)
def _root_definitions(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
for defs_key in ("$defs", "definitions"):
yield from ((f"#/{defs_key}/{name}", body) for name, body in _named_schemas(_schema_at(schema, defs_key)))
def _refs_in(schema: Mapping[str, object]) -> frozenset[str]:
return frozenset(
schema_fragment_ref(ref) for node in _schema_nodes(schema) if isinstance(ref := node.get("$ref"), str)
)
def _property_schemas(schema: Mapping[str, object]) -> Iterator[tuple[str, Mapping[str, object]]]:
return chain.from_iterable(_named_schemas(_schema_at(node, "properties")) for node in _schema_nodes(schema))
def _is_union(node: Mapping[str, object]) -> bool:
return isinstance(node.get("anyOf"), list) or isinstance(node.get("oneOf"), list) or len(_list_at(node, "type")) > 1
def _grammar_branches(node: Mapping[str, object]) -> int:
properties: Final = _schema_at(node, "properties")
if properties is None:
return 0
optional: Final = len(_optional_property_names(node))
return 2**optional * (len(properties) - optional + 1)
def _optional_property_names(node: Mapping[str, object]) -> frozenset[str]:
properties: Final = _schema_at(node, "properties")
if properties is None:
return frozenset()
return frozenset(properties) - frozenset(name for name in _list_at(node, "required") if isinstance(name, str))
def _is_open_object(node: Mapping[str, object]) -> bool:
additional_properties: Final = node.get("additionalProperties")
if additional_properties is True or isinstance(additional_properties, dict) or "patternProperties" in node:
return True
return is_object_schema(node) and "properties" not in node and additional_properties is not False
def _subschemas(node: Mapping[str, object]) -> Iterator[Mapping[str, object]]:
for key, value in node.items():
if key in _SUBSCHEMA_MAP_KEYWORDS and isinstance(value, dict):
@ -1894,6 +2012,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa
return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS)
def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
"""Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys."""
return _message_without_fields(message, frozenset({"prefix"}))
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
"""
Filters a value from a dictionary

34
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -4303,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag
return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]])
def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock:
return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block
def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock:
"""Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it."""
if message["role"] != "assistant":
return message
blocks: Final = message["content"]
last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None)
return BedrockMessageBlock(
role="assistant",
content=[
_rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks)
],
)
def _ensure_document_messages_have_text(
contents: list[BedrockMessageBlock],
) -> list[BedrockMessageBlock]:
@ -4894,7 +4912,12 @@ class BedrockConverseMessagesProcessor:
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(
_rename_duplicate_bedrock_document_names(contents)
)
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
@staticmethod
def translate_thinking_blocks_to_reasoning_content_blocks(
@ -5303,7 +5326,10 @@ def _bedrock_converse_messages_pt(
llm_provider=llm_provider,
)
return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
if not (to_claude and converse_messages):
return converse_messages
return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
@ -5478,14 +5504,14 @@ def _bedrock_tools_pt(
description = name
defs = parameters.pop("$defs", {})
defs_copy = copy.deepcopy(defs)
defs_copy = copy.deepcopy(defs) if isinstance(defs, dict) else {}
# Expand $ref references in parameters using the definitions
# Note: We don't pre-flatten defs as that causes exponential memory growth
# with circular references (see issue #19098). unpack_defs handles nested
# refs recursively and correctly detects/skips circular references.
unpack_defs(parameters, defs_copy)
normalize_json_schema_custom_types_to_object(parameters)
if parameters.get("type") not in _valid_json_schema_root_types:
if not (isinstance(root_type := parameters.get("type"), str) and root_type in _valid_json_schema_root_types):
parameters["type"] = "object"
tool_block = cast(
BedrockToolBlock,

247
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -1,17 +1,20 @@
import asyncio
import json
import traceback
from collections.abc import Coroutine, Mapping, Sequence
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable
from pydantic import TypeAdapter
from typing_extensions import ReadOnly
import litellm
from litellm._logging import redact_internal_details_from_client_message, verbose_logger
from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
REALTIME_METERED_USAGE_KEY,
REALTIME_SESSION_FAILURE_LOGGED_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
@ -98,6 +101,16 @@ class ScopedWebSocket(Protocol):
def scope(self) -> _ASGIScope: ...
@runtime_checkable
class _ReplayScopeWebSocket(Protocol):
@property
def scope(self) -> MutableMapping[str, object]: ...
def client_websocket_scope(websocket: object) -> MutableMapping[str, object] | None:
return websocket.scope if isinstance(websocket, _ReplayScopeWebSocket) else None
class _ClientWebSocket(ScopedWebSocket, Protocol):
async def send_text(self, data: str) -> None: ...
async def receive_text(self) -> str: ...
@ -116,10 +129,87 @@ def _decode_json_object(payload: str) -> Mapping[str, object]:
return json.loads(payload)
_GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType(
{"type": "server_vad", "create_response": False}
)
_EMPTY_SESSION: Final[Mapping[str, object]] = MappingProxyType({})
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_GUARDRAIL_EVENT_ID_PREFIX: Final = "litellm_guardrail_"
def _json_object(value: object) -> Mapping[str, object] | None:
return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
return _json_object(container.get(key))
def _guardrail_event_id() -> str:
return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
def _error_event_id(event: Mapping[str, object]) -> str | None:
error: Final = _mapping_field(event, "error")
caused_by: Final = None if error is None else error.get("event_id")
return caused_by if isinstance(caused_by, str) else None
_TurnDetectionGate: TypeAlias = Callable[[object], Mapping[str, object]]
def _turn_detection_with_auto_response_off(turn_detection: object) -> Mapping[str, object]:
client_turn_detection: Final = _json_object(turn_detection)
if client_turn_detection is None:
return dict(_GUARDRAIL_TURN_DETECTION_GATE)
return {**client_turn_detection, "create_response": False}
def _nested_turn_detection_part(session: Mapping[str, object], gate: _TurnDetectionGate) -> Mapping[str, object]:
audio: Final = _mapping_field(session, "audio")
if audio is None:
return {}
audio_input: Final = _mapping_field(audio, "input")
if audio_input is None or "turn_detection" not in audio_input:
return {}
gated_input: Final = {
**audio_input,
"turn_detection": gate(audio_input["turn_detection"]),
}
return {"audio": {**audio, "input": gated_input}}
def _with_create_response_disabled(
session: Mapping[str, object], gate: _TurnDetectionGate
) -> Mapping[str, object] | None:
"""``session`` with every turn_detection the client sent unable to auto-respond, or None when it sent none."""
flat_part: Final = {"turn_detection": gate(session["turn_detection"])} if "turn_detection" in session else {}
nested_part: Final = _nested_turn_detection_part(session, gate)
if not flat_part and not nested_part:
return None
return {**session, **flat_part, **nested_part}
def _gate_session_update(
session: object, gate: _TurnDetectionGate, inject_gate_when_absent: bool
) -> Mapping[str, object] | None:
"""The session to forward under a transcript guardrail, or None when the frame needs no change."""
client_session: Final = _json_object(session)
if client_session is None:
return None
gated: Final = _with_create_response_disabled(client_session, gate)
if gated is not None or not inject_gate_when_absent:
return gated
return {**client_session, "turn_detection": gate(None)}
class RealtimeEventNormalizer(Protocol):
def should_drop(self, event: object) -> bool: ...
def normalize(self, event: dict) -> dict: ...
def patch_outgoing_session(self, session: dict) -> dict: ...
def gate_turn_detection(self, turn_detection: object, /) -> Mapping[str, object]: ...
@property
def deletes_conversation_items(self) -> bool: ...
class RealtimeUsageMeter(Protocol):
@ -154,6 +244,7 @@ class RealTimeStreaming:
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
):
self.websocket: _ClientWebSocket = websocket
self._replay_scope: Final = client_websocket_scope(websocket)
self.backend_ws = backend_ws
self.logging_obj = logging_obj
self._logging_worker = logging_worker
@ -197,6 +288,8 @@ class RealTimeStreaming:
# Track whether we have already sent the guardrail turn-detection update
# that disables provider auto-response for transcription guardrails.
self._guardrail_turn_detection_update_sent: bool = False
self._session_gate_sent: bool = False
self._guardrail_cancel_event_ids: frozenset[str] = frozenset()
# Deferred Gemini Live setup: Pipecat may stream audio before session.update.
# Buffer client audio until the backend acknowledges setup (setupComplete).
self._backend_setup_complete: bool = provider_config is None or provider_config.requires_session_configuration()
@ -213,6 +306,12 @@ class RealTimeStreaming:
self._is_transcription_session: bool = force_transcription_model is not None
# Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
self._event_normalizer = event_normalizer
self._gate_turn_detection: Final[_TurnDetectionGate] = (
_turn_detection_with_auto_response_off if event_normalizer is None else event_normalizer.gate_turn_detection
)
self._deletes_conversation_items: Final = (
event_normalizer is None or event_normalizer.deletes_conversation_items
)
self._usage_meter = usage_meter
@property
@ -734,20 +833,37 @@ class RealTimeStreaming:
except (json.JSONDecodeError, TypeError):
return
def _make_turn_detection_update(self, turn_detection: Mapping[str, object]) -> dict[str, object]:
session: Final[dict[str, object]] = (
{"turn_detection": turn_detection}
if self._backend_uses_beta_protocol
else {"type": "realtime", "audio": {"input": {"turn_detection": turn_detection}}}
)
return {"type": "session.update", "session": session}
def _make_disable_auto_response_message(self) -> str:
"""Return a session.update that disables VAD auto-response."""
turn_detection: Final[dict[str, str | bool]] = {
"type": "server_vad",
"create_response": False,
}
if self._backend_uses_beta_protocol:
session: dict[str, object] = {"turn_detection": turn_detection}
else:
session = {
"type": "realtime",
"audio": {"input": {"turn_detection": turn_detection}},
}
return json.dumps({"type": "session.update", "session": session})
return json.dumps(self._make_turn_detection_update(self._gate_turn_detection(None)))
async def _send_guardrail_cancel(self) -> None:
event_id: Final = _guardrail_event_id()
self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
await self._send_guardrail_cancel()
if item_id is not None and self._deletes_conversation_items:
await self._send_to_backend(json.dumps({"type": "conversation.item.delete", "item_id": item_id}))
def _is_guardrail_cancel_error(self, event: Mapping[str, object]) -> bool:
return _error_event_id(event) in self._guardrail_cancel_event_ids
async def _send_session_gate_once(self) -> None:
"""Gate the backend's default turn_detection ahead of the first client session.update, which overrides it."""
if self._session_gate_sent or not self._should_disable_vad_auto_response():
return
self._session_gate_sent = True
await self._send_to_backend(self._make_disable_auto_response_message())
async def _maybe_send_guardrail_turn_detection_update(self) -> None:
"""Disable provider auto-response once when transcription guardrails are enabled."""
@ -921,7 +1037,7 @@ class RealTimeStreaming:
if pre_block_backend_message is not None:
await self._send_to_backend(pre_block_backend_message)
if not self._is_transcription_session:
await self._send_to_backend(json.dumps({"type": "response.cancel"}))
await self._withdraw_blocked_turn(item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
@ -1051,10 +1167,7 @@ class RealTimeStreaming:
self.store_message(event_str)
self._capture_transcription_usage(event)
await self._send_event_to_client(event, event_str)
blocked = await self.run_realtime_guardrails(
cast(str, transcript),
item_id=cast(str | None, event.get("item_id")),
)
blocked = await self.run_realtime_guardrails(cast(str, transcript))
if not blocked and self._should_disable_vad_auto_response():
await self._send_to_backend(json.dumps({"type": "response.create"}))
continue
@ -1086,7 +1199,7 @@ class RealTimeStreaming:
if event_type == "session.created" and self._should_disable_vad_auto_response():
self.store_message(event_obj)
await self.websocket.send_text(self._event_to_client_json(event_obj))
await self._send_to_backend(self._make_disable_auto_response_message())
await self._send_session_gate_once()
return True
if event_type == "conversation.item.input_audio_transcription.completed":
@ -1140,6 +1253,10 @@ class RealTimeStreaming:
if self._usage_meter is not None:
self._usage_meter.observe_backend_event(event)
if event.get("type") == "error" and self._is_guardrail_cancel_error(event):
verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
continue
if await self._handle_raw_backend_message(event, raw_response):
continue
@ -1346,6 +1463,13 @@ class RealTimeStreaming:
return item
async def _receive_client_message(self) -> str | None:
refused_opening_message: Final = (
None
if self._replay_scope is None
else self._replay_scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
)
if isinstance(refused_opening_message, str):
return refused_opening_message
try:
return await self.websocket.receive_text()
except Exception as e: # noqa: BLE001 # whatever the client socket raises, the client is gone
@ -1472,19 +1596,21 @@ class RealTimeStreaming:
and self.session_configuration_request is None
and not self._guardrail_turn_detection_update_sent
and self._should_disable_vad_auto_response()
):
session: Mapping[str, object] | None = msg_obj.setdefault("session", {})
if isinstance(session, dict):
existing_td = session.get("turn_detection")
if not isinstance(existing_td, dict):
existing_td = {}
existing_td["create_response"] = False
session["turn_detection"] = existing_td
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
and (
gated_first := _gate_session_update(
client_event.get("session", _EMPTY_SESSION),
self._gate_turn_detection,
inject_gate_when_absent=True,
)
)
is not None
):
msg_obj["session"] = gated_first
message = json.dumps(msg_obj)
guardrail_turn_detection_injected = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
)
## GUARDRAIL: Force ``create_response`` to False in any
# client-provided ``turn_detection`` so a later
@ -1492,52 +1618,25 @@ class RealTimeStreaming:
# and bypass the transcription guardrail after the
# initial disable. Covers both the flat beta key and the
# nested GA ``audio.input.turn_detection`` shape, since
# the GA remap below also accepts either form. Skipped
# when the injection block above already ran for this
# message, to avoid redundant double-serialization.
# the GA remap below also accepts either form. A later
# update with no turn_detection is forwarded as is: OpenAI
# keeps the earlier disable for an unset field, and xAI
# falls back to its manual default, which never auto-responds.
if (
msg_type == "session.update"
and not guardrail_turn_detection_injected
and self._should_disable_vad_auto_response()
and (
gated_later := _gate_session_update(
client_event.get("session"),
self._gate_turn_detection,
inject_gate_when_absent=False,
)
)
is not None
):
session = client_event.get("session")
if isinstance(session, dict):
td_overridden = False
flat_td = session.get("turn_detection")
flat_td_present = flat_td is not None
if flat_td_present:
if not isinstance(flat_td, dict):
flat_td = {}
if flat_td.get("create_response") is not False:
flat_td["create_response"] = False
session["turn_detection"] = flat_td
td_overridden = True
nested_td_present = False
audio = session.get("audio")
if isinstance(audio, dict):
audio_input = audio.get("input")
if isinstance(audio_input, dict):
nested_td = audio_input.get("turn_detection")
if nested_td is not None:
nested_td_present = True
if not isinstance(nested_td, dict):
nested_td = {}
if nested_td.get("create_response") is not False:
nested_td["create_response"] = False
audio_input["turn_detection"] = nested_td
td_overridden = True
# Symmetric with the first-update injection block:
# if the client omitted turn_detection entirely on
# a subsequent session.update, still inject the
# ``create_response: False`` override so the
# transcription guardrail cannot be re-enabled by
# any downstream merge that drops the original
# disable.
if not flat_td_present and not nested_td_present:
session["turn_detection"] = {"create_response": False}
td_overridden = True
if td_overridden:
message = json.dumps(msg_obj)
msg_obj["session"] = gated_later
message = json.dumps(msg_obj)
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
@ -1590,6 +1689,8 @@ class RealTimeStreaming:
# would permanently disable the injection if ``_send_to_backend``
# raised — neither this loop nor
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
if msg_type == "session.update" and self.provider_config is None:
await self._send_session_gate_once()
sent = await self._send_to_backend(message)
if guardrail_turn_detection_injected and sent:
self._guardrail_turn_detection_update_sent = True

96
litellm/litellm_core_utils/streaming_handler.py
View file

@ -242,6 +242,7 @@ class CustomStreamWrapper:
self.completion_stream = completion_stream
self.sent_first_chunk = False
self.sent_last_chunk = False
self._failure_logging: asyncio.Task[None] | None = None
self._closed = False
self._stream_created_time: float = time.time()
@ -269,6 +270,7 @@ class CustomStreamWrapper:
self.holding_chunk = ""
self.complete_response = ""
self.response_uptil_now = ""
self._emitted_disqualifying_content = False
_model_info: Final[dict] = litellm_params.model_info or {}
_api_base: Final = get_api_base(
@ -1126,31 +1128,24 @@ class CustomStreamWrapper:
"""
if self.merge_reasoning_content_in_choices is True:
reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None)
if reasoning_content:
if self.sent_first_thinking_block is False:
# Ensure content is not None before concatenation
if model_response.choices[0].delta.content is None:
model_response.choices[0].delta.content = ""
model_response.choices[0].delta.content += "<think>" + reasoning_content
self.sent_first_thinking_block = True
elif (
self.sent_first_thinking_block is True
and hasattr(model_response.choices[0].delta, "reasoning_content")
and model_response.choices[0].delta.reasoning_content
):
model_response.choices[0].delta.content = reasoning_content
elif (
self.sent_first_thinking_block is True
and not self.sent_last_thinking_block
and model_response.choices[0].delta.content
):
model_response.choices[0].delta.content = "</think>" + (model_response.choices[0].delta.content or "")
self.sent_last_thinking_block = True
if self.merge_reasoning_content_in_choices is not True:
return
delta: Final = model_response.choices[0].delta
reasoning_content: Final = getattr(delta, "reasoning_content", None)
content: Final = delta.content or ""
if reasoning_content:
self._emitted_disqualifying_content = True
closes_thinking: Final = bool(content) and not self.sent_last_thinking_block
opening: Final = "" if self.sent_first_thinking_block else "<think>"
delta.content = opening + reasoning_content + ("</think>" + content if closes_thinking else "")
self.sent_first_thinking_block = True
self.sent_last_thinking_block = self.sent_last_thinking_block or closes_thinking
elif self.sent_first_thinking_block and not self.sent_last_thinking_block and content:
delta.content = "</think>" + content
self.sent_last_thinking_block = True
if hasattr(model_response.choices[0].delta, "reasoning_content"):
del model_response.choices[0].delta.reasoning_content
if hasattr(delta, "reasoning_content"):
del delta.reasoning_content
def _dispatch_provider_chunk(
self,
@ -1865,9 +1860,7 @@ class CustomStreamWrapper:
if response.choices:
choice = response.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# HANDLE STREAM OPTIONS
self.chunks.append(response)
@ -2084,9 +2077,7 @@ class CustomStreamWrapper:
if processed_chunk.choices:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# Add mcp_list_tools to first chunk if present
if not self.sent_first_chunk and processed_chunk.choices:
@ -2156,9 +2147,7 @@ class CustomStreamWrapper:
choice = processed_chunk.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
else:
self.response_uptil_now += ""
self._accumulate_streamed_delta(choice.delta)
self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
# RETURN RESULT
self.chunks.append(processed_chunk)
@ -2172,7 +2161,7 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
await self.aclose()
@ -2305,11 +2294,17 @@ class CustomStreamWrapper:
if self.logging_obj is not None:
self._record_partial_usage_for_failure()
## LOGGING
asyncio.create_task(
self._failure_logging = asyncio.create_task(
self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
)
self._handle_stream_fallback_error(e)
async def wait_for_failure_logging(self, timeout: float) -> None:
"""Let this stream's failure log read its own deployment before a fallback reuses the logging object."""
if self._failure_logging is None:
return
await asyncio.wait((self._failure_logging,), timeout=timeout)
def _record_partial_usage_for_failure(self) -> None:
"""
A stream that breaks mid-flight still billed the provider for the chunks
@ -2346,6 +2341,35 @@ class CustomStreamWrapper:
recover_error,
)
# Delta fields a text-only prefill continuation cannot carry, so a stream
# that emitted any of them is not eligible for mid-stream continuation.
_CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = (
"tool_calls",
"function_call",
"reasoning_content",
"thinking_blocks",
"reasoning_items",
"audio",
"images",
"annotations",
)
@classmethod
def _delta_disqualifies_continuation(cls, delta: object) -> bool:
get: Final = getattr(delta, "get", None)
if not callable(get):
return False
return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
def _accumulate_streamed_delta(self, delta: object) -> None:
# Shared by the sync, async, and non-aiohttp iteration sites so answer
# text and the disqualifying-content latch stay in step across all three.
get: Final = getattr(delta, "get", None)
content: Final = get("content") if callable(get) else None
self.response_uptil_now += content if isinstance(content, str) else ""
if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
self._emitted_disqualifying_content = True
def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
"""
Common error handling for both __next__ and __anext__.
@ -2377,6 +2401,7 @@ class CustomStreamWrapper:
original_exception=e.original_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content,
)
# Map to OpenAI exception format. Some providers' mappers (e.g.
@ -2437,6 +2462,7 @@ class CustomStreamWrapper:
original_exception=mapped_exception,
generated_content=self.response_uptil_now,
is_pre_first_chunk=not self.sent_first_chunk,
emitted_disqualifying_content=self._emitted_disqualifying_content,
)
@staticmethod

164
litellm/litellm_core_utils/tool_fields.py Normal file
View file

@ -0,0 +1,164 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import reduce
from typing import Annotated, Final, TypeAlias
from pydantic import BeforeValidator, ConfigDict, Json, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
ROUTES_WITH_TOOLS: Final = frozenset(
{
"acompletion",
"aresponses",
"acompact_responses",
"anthropic_messages",
"agenerate_content",
"agenerate_content_stream",
"acreate_interaction",
}
)
TOOL_FIELDS: Final = frozenset({"tools", "tool_choice", "parallel_tool_calls", "toolConfig", "tool_config"})
_TOOLS: Final = TypeAdapter(list[object])
_FIELD_LIST: Final = TypeAdapter(list[object], config=ConfigDict(strict=True))
_JSON_OBJECT: Final = TypeAdapter(Json[dict[str, object]])
def _schema_as_read(value: object) -> object:
if not value:
return None
if not isinstance(value, str):
return value
try:
return _JSON_OBJECT.validate_python(value)
except ValidationError:
return value
_Schema: TypeAlias = Annotated[dict[str, object] | None, BeforeValidator(_schema_as_read)]
class _ToolFunction(TypedDict, total=False):
parameters: ReadOnly[_Schema]
class _ToolEntry(TypedDict, total=False):
function: ReadOnly[_ToolFunction | None]
parameters: ReadOnly[_Schema]
input_schema: ReadOnly[_Schema]
vector_store_ids: ReadOnly[list[str] | None]
_TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
_WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
@dataclass(frozen=True, slots=True)
class DropPolicy:
drop_params: bool
additional_drop_params: frozenset[str]
def drops(self, field: str) -> bool:
return self.drop_params or field in self.additional_drop_params
@dataclass(frozen=True, slots=True)
class MalformedField:
field: str
message: str
dropped: Callable[[Mapping[str, object]], Mapping[str, object]]
def _field_names(value: object) -> frozenset[str]:
try:
return frozenset(name for name in _FIELD_LIST.validate_python(value) if isinstance(name, str))
except ValidationError:
return frozenset()
def drop_policy(params: Mapping[str, object]) -> DropPolicy:
return DropPolicy(
drop_params=litellm.drop_params is True or normalize_drop_params(params.get("drop_params")) is True,
additional_drop_params=_field_names(params.get("additional_drop_params")),
)
def first_validation_error(e: ValidationError, prefix: tuple[str | int, ...] = ()) -> str:
error: Final = e.errors(include_url=False)[0]
return f"{'.'.join(str(part) for part in (*prefix, *error['loc']))}: {error['msg']}"
def _tool_entry_error(index: int, entry: object) -> str | None:
try:
_TOOL_ENTRY.validate_python(entry, strict=True)
except ValidationError as e:
return first_validation_error(e, prefix=("tools", index))
return None
def _without_tool_fields(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name not in TOOL_FIELDS}
def _malformed_tools(data: Mapping[str, object]) -> MalformedField | None:
tools: Final = data.get("tools")
if tools is None:
return None
try:
entries: Final = _TOOLS.validate_python(tools)
except ValidationError as e:
return MalformedField("tools", first_validation_error(e, prefix=("tools",)), _without_tool_fields)
entry_errors: Final = tuple(_tool_entry_error(index, entry) for index, entry in enumerate(entries))
first_error: Final = next((error for error in entry_errors if error is not None), None)
if first_error is None:
return None
kept: Final = [entry for entry, error in zip(entries, entry_errors) if error is None]
return MalformedField(
"tools", first_error, lambda fields: {**fields, "tools": kept} if kept else _without_tool_fields(fields)
)
def _without_web_search_options(data: Mapping[str, object]) -> Mapping[str, object]:
return {name: value for name, value in data.items() if name != "web_search_options"}
def _malformed_web_search_options(data: Mapping[str, object]) -> MalformedField | None:
try:
_WEB_SEARCH_OPTIONS.validate_python(data.get("web_search_options"))
except ValidationError as e:
return MalformedField(
"web_search_options",
first_validation_error(e, prefix=("web_search_options",)),
_without_web_search_options,
)
return None
def resolve_tool_fields(
data: Mapping[str, object], candidates: Callable[[], tuple[DropPolicy, ...]]
) -> Mapping[str, object] | MalformedField:
checked: Final = (_malformed_tools(data), _malformed_web_search_options(data))
malformed: Final = tuple(field for field in checked if field is not None)
if not malformed:
return data
policies: Final = candidates()
rejected: Final = next(
(field for field in malformed if not any(policy.drops(field.field) for policy in policies)), None
)
if rejected is not None:
return rejected
return reduce(
lambda resolved, field: field.dropped(resolved),
(field for field in malformed if all(policy.drops(field.field) for policy in policies)),
data,
)
def resolve_deployment_tool_fields(
kwargs: Mapping[str, object], route: object
) -> Mapping[str, object] | MalformedField:
if "proxy_server_request" not in kwargs or route not in ROUTES_WITH_TOOLS:
return kwargs
return resolve_tool_fields(kwargs, lambda: (drop_policy(kwargs),))

5
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -45,6 +45,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
scoped_structured_message_indices,
@ -924,10 +925,10 @@ class AnthropicMessagesHandler(BaseTranslation):
images=(),
)
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract every tool name in an Anthropic messages request: tools[].name, plus
tools[].function.name for OpenAI-format tools the bridge forwards verbatim."""
return [name for tool in data.get("tools") or [] for name in anthropic_tool_names(tool)]
return [name for tool in list_field(data, "tools") for name in anthropic_tool_names(tool)]
@classmethod
def _extract_input_text_and_images(

114
litellm/llms/anthropic/chat/transformation.py
View file

@ -1,3 +1,4 @@
import copy
import json
import re
import time
@ -24,7 +25,18 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
is_object_schema,
sanitize_input_schema_for_anthropic,
schema_fragment_ref,
schema_grammar_branch_count,
schema_has_lookaround_pattern,
schema_has_open_object,
schema_has_unresolvable_ref,
schema_optional_property_count,
schema_string_formats,
schema_union_property_count,
unpack_defs,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
RemoteMedia,
@ -43,7 +55,7 @@ from litellm.llms.base_llm.base_utils import (
type_to_response_format_param,
unwrap_response_format_envelope,
)
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
@ -150,6 +162,12 @@ _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
_DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
NATIVE_OUTPUT_MAX_UNION_PARAMETERS: Final = 16
NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24
NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048
def _tool_type(tool: object) -> str | None:
@ -753,7 +771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
grammar Anthropic compiles the schema into, so the API rejects them with a
400 ``invalid_request_error`` (e.g. "output_format.schema: For 'array' type,
property 'uniqueItems' is not supported"):
- maxItems/minItems/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- maxItems/minItems above 1/uniqueItems/contains/minContains/maxContains/prefixItems: array constraints
- minimum/maximum/exclusiveMinimum/exclusiveMaximum/multipleOf: numeric constraints
- minLength/maxLength: string constraints
- minProperties/maxProperties/patternProperties/propertyNames: object constraints
@ -810,7 +828,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"else": "conditional (else): {}",
"not": "must not match: {}",
}
unsupported_fields: Final = set(constraint_labels)
min_items: Final = schema.get("minItems")
enforced_min_items: Final = (
isinstance(min_items, int) and not isinstance(min_items, bool) and min_items in (0, 1)
)
unsupported_fields: Final = set(constraint_labels) - ({"minItems"} if enforced_min_items else set())
# Build description additions from removed constraints. Iterating
# constraint_labels (not the set) keeps the note order deterministic across
@ -818,7 +840,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# PYTHONHASHSEED and stay cache-friendly.
constraint_descriptions: Final[list] = []
for field, label in constraint_labels.items():
if field not in schema:
if field not in schema or field not in unsupported_fields:
continue
value = schema[field]
# A falsy boolean constraint (e.g. ``uniqueItems: false``) imposes no
@ -853,7 +875,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Already handled above
continue
if key in ("properties", "$defs") and isinstance(value, dict):
if key in ("properties", "$defs", "definitions") and isinstance(value, dict):
nested_schemas: dict[str, Any] = value # mutable-ok: filter_anthropic_output_schema takes a dict
result[key] = {k: AnthropicConfig.filter_anthropic_output_schema(v) for k, v in nested_schemas.items()}
elif key == "items" and isinstance(value, dict):
@ -865,6 +887,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
]
elif key == "allOf" and isinstance(value, list):
result[key] = [AnthropicConfig.filter_anthropic_output_schema(item) for item in value]
elif key == "$ref" and isinstance(value, str):
result[key] = schema_fragment_ref(value)
elif key == "oneOf" and isinstance(value, list):
# Anthropic rejects oneOf ("Schema type 'oneOf' is not supported");
# the Anthropic SDK rewrites it to anyOf, so do the same.
@ -876,7 +900,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Anthropic requires additionalProperties=false for object schemas
# See: https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs
if result.get("type") == "object":
if is_object_schema(result):
result["additionalProperties"] = False
return result
@ -1628,32 +1652,55 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
return json_schema
@staticmethod
def _defs_unpacked(json_schema: Mapping[str, object]) -> Mapping[str, object]:
unpacked: Final = copy.deepcopy(dict(json_schema))
defs: Final = unpacked.pop("$defs", unpacked.pop("definitions", None))
if isinstance(defs, dict) and defs:
unpack_defs(unpacked, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
return unpacked
@staticmethod
def native_output_schema(json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
if (
schema_has_unresolvable_ref(json_schema)
or schema_has_open_object(json_schema)
or schema_has_lookaround_pattern(json_schema)
or not schema_string_formats(json_schema) <= _NATIVE_OUTPUT_STRING_FORMATS
or schema_union_property_count(json_schema) > NATIVE_OUTPUT_MAX_UNION_PARAMETERS
or schema_optional_property_count(json_schema) > NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS
or schema_grammar_branch_count(json_schema) > NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES
):
return None
return AnthropicConfig.filter_anthropic_output_schema(dict(json_schema))
def uses_native_structured_output_for_schema(
self,
model: str,
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
open_object_takes_forced_json_tool: Final = (
json_schema is not None
and schema_has_open_object(json_schema)
and not self.is_thinking_requested(request_params)
and not self.is_thinking_enabled(dict(request_params))
and not request_params.get("tools")
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
)
return self.uses_native_structured_output(model) and not open_object_takes_forced_json_tool
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return self.filter_anthropic_output_schema(dict(self._defs_unpacked(json_schema)))
def map_response_format_to_anthropic_output_format(self, value: dict | None) -> AnthropicOutputSchema | None:
json_schema: dict | None = self._extract_json_schema_from_response_format(value)
json_schema: Final = self._extract_json_schema_from_response_format(value)
if json_schema is None:
return None
# Resolve $ref/$defs before filtering — Anthropic doesn't support
# external schema references (e.g., /$defs/CalendarEvent).
import copy
from litellm.litellm_core_utils.prompt_templates.common_utils import (
DEFS_MAX_INLINED_BYTES,
unpack_defs,
)
json_schema = copy.deepcopy(json_schema)
defs: Final = json_schema.pop("$defs", json_schema.pop("definitions", {}))
if defs:
unpack_defs(json_schema, defs, max_inlined_bytes=DEFS_MAX_INLINED_BYTES)
# Filter out unsupported fields for Anthropic's output_format API
filtered_schema: Final = self.filter_anthropic_output_schema(json_schema)
return AnthropicOutputSchema(
type="json_schema",
schema=filtered_schema,
)
native_schema: Final = self.native_output_format_schema(json_schema)
if native_schema is None:
return None
return AnthropicOutputSchema(type="json_schema", schema=dict(native_schema))
def map_response_format_to_anthropic_tool(
self, value: dict | None, optional_params: dict, is_thinking_enabled: bool
@ -1820,7 +1867,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_key=param,
)
elif param == "response_format" and isinstance(value, dict):
if self.uses_native_structured_output(model):
if self.uses_native_structured_output_for_schema(
model,
self._extract_json_schema_from_response_format(value),
REQUEST_PARAMS.validate_python(non_default_params),
):
_output_format = self.map_response_format_to_anthropic_output_format(value)
if _output_format is not None:
optional_params["output_format"] = _output_format
@ -2979,6 +3030,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
completion_response["stop_reason"] = "stop"
if refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks
json_mode_message.reasoning_content = reasoning_content
_message = json_mode_message
model_response.choices[0].message = _message

3
litellm/llms/anthropic/count_tokens/token_counter.py
View file

@ -11,7 +11,7 @@ from litellm._logging import verbose_logger
from litellm.exceptions import AuthenticationError
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler()
@ -37,6 +37,7 @@ class AnthropicTokenCounter(BaseTokenCounter):
system: JsonValue = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Anthropic's CountTokens API.

41
litellm/llms/anthropic/pass_through/adapters/handler.py
View file

@ -139,7 +139,7 @@ def request_safeguards_evaluator(
)
async def _prepare_context_managed_request(
async def prepare_context_managed_request(
*,
model: str,
messages: _AnthropicMessages,
@ -152,6 +152,8 @@ async def _prepare_context_managed_request(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
drop_invalid_edits: bool = False,
) -> PolyfillResult | None:
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
@ -193,6 +195,8 @@ async def _prepare_context_managed_request(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
drop_invalid_edits=drop_invalid_edits,
)
if polyfill_result is not None:
@ -266,16 +270,14 @@ def _spec_has_non_compact_edits(
return any(isinstance(edit.get("type"), str) and edit.get("type") != COMPACT_EDIT_TYPE for edit in edits)
def _context_management_explicitly_dropped(additional_drop_params: list[str] | None) -> bool:
def context_management_explicitly_dropped(additional_drop_params: object) -> bool:
"""True when the caller opted out of context_management via ``additional_drop_params``.
``drop_params`` deliberately does NOT gate the polyfill: ``context_management``
is a LiteLLM-supported param (native on Anthropic, polyfilled elsewhere), and
``drop_params`` only exists to drop genuinely unsupported params.
"""
if not isinstance(additional_drop_params, list):
return False
return "context_management" in additional_drop_params
return isinstance(additional_drop_params, list) and "context_management" in additional_drop_params
def _normalize_spec_edits(
@ -291,7 +293,7 @@ def _normalize_spec_edits(
if not context_management_spec:
return None
if _context_management_explicitly_dropped(additional_drop_params):
if context_management_explicitly_dropped(additional_drop_params):
return None
from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
@ -317,20 +319,23 @@ async def _run_polyfill_if_enabled(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
drop_invalid_edits: bool = False,
) -> PolyfillResult | None:
"""Run the async context_management polyfill if a spec is present.
Returns ``None`` when the spec is empty or ``context_management`` is
listed in ``additional_drop_params`` (the explicit opt-out; ``drop_params``
does not disable the polyfill because context_management is a supported
does not disable a valid spec because context_management is a supported
param). Raises ``AnthropicContextManagementError`` so the /v1/messages
endpoint can emit an Anthropic-format 400. All other exceptions are
endpoint can emit an Anthropic-format 400, unless ``drop_invalid_edits``
(``drop_params``) drops the invalid spec. All other exceptions are
best-effort swallowed (matches v0 behavior).
"""
if not context_management_spec:
return None
if _context_management_explicitly_dropped(additional_drop_params):
if context_management_explicitly_dropped(additional_drop_params):
return None
try:
@ -345,13 +350,17 @@ async def _run_polyfill_if_enabled(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
)
except AnthropicContextManagementError:
if drop_invalid_edits:
return None
# Surface validation errors so the endpoint can emit an Anthropic-format
# 400. Other exception types fall into the best-effort branch below.
raise
except Exception as e:
verbose_logger.exception("context_management polyfill: skipping edits due to error: %s", e)
log_failure: Final = verbose_logger.debug if count_only else verbose_logger.exception
log_failure("context_management polyfill: skipping edits due to error: %s", e)
# Best-effort swallow is only safe for compact-only specs, where the
# caller's compaction-block-slicing safety net produces a correct
# (if degraded) result. When the spec also requested non-compact
@ -709,6 +718,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
"""Handle non-Anthropic models asynchronously using the adapter"""
context_management: Final = kwargs.pop("context_management", None)
additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
drop_invalid_edits: Final = litellm.drop_params is True or (
"drop_params" in kwargs and kwargs["drop_params"] is True
)
litellm_router: Final[Router | None] = _router_or_proxy_fallback(kwargs.pop("litellm_router", None))
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
@ -716,7 +728,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs)
sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs)
polyfill_result: Final = await _prepare_context_managed_request(
polyfill_result: Final = await prepare_context_managed_request(
model=model,
messages=messages,
tools=tools,
@ -728,6 +740,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=sub_call_trace_id,
drop_invalid_edits=drop_invalid_edits,
)
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages
@ -841,6 +854,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
# bridge to it via ``run_async_function``.
context_management: Final = kwargs.pop("context_management", None)
additional_drop_params: Final[list[str] | None] = kwargs.get("additional_drop_params", None)
drop_invalid_edits: Final = litellm.drop_params is True or (
"drop_params" in kwargs and kwargs["drop_params"] is True
)
# Deliberately do NOT auto-attach the proxy ``llm_router`` here:
# ``run_async_function`` spawns a new event loop in a worker thread
# to bridge to the async dispatcher, but the proxy router's httpx
@ -866,7 +882,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs)
polyfill_result = run_async_function(
_prepare_context_managed_request,
prepare_context_managed_request,
model=model,
messages=messages,
tools=tools,
@ -878,6 +894,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=proxy_sub_call_logging_kwargs(request_kwargs),
parent_trace_id=proxy_sub_call_trace_id(request_kwargs),
drop_invalid_edits=drop_invalid_edits,
)
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages

75
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py
View file

@ -24,6 +24,7 @@ from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
@ -39,6 +40,7 @@ from litellm.types.llms.anthropic import (
UsageDelta,
UsageIteration,
)
from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDeltaToolCall, Delta, Function
if TYPE_CHECKING:
@ -50,6 +52,7 @@ if TYPE_CHECKING:
_STREAMING_DELTA_TYPES: Final = frozenset(get_args(StreamingContentBlockDeltaType))
_SSE_EVENT: Final = TypeAdapter(Mapping[str, object])
_THINKING_BLOCK: Final = TypeAdapter(ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock)
class _UsageDeltaWithIterations(UsageDelta, total=False):
@ -106,6 +109,14 @@ def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
assert_never(delta_type)
def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None:
delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
first_block: Final = as_str_mapping(blocks[0]) if blocks else None
index: Final = first_block.get("index") if first_block is not None else None
return index if isinstance(index, int) else None
def _optional_str_attr(obj: object, name: str) -> str | None:
value: Final = _optional_attr(obj, name)
return value if isinstance(value, str) else None
@ -373,13 +384,44 @@ class _CombinedChunkSplitter:
finish_delta.thinking_blocks = None
return [content_chunk, finish_chunk]
@staticmethod
def _with_thinking_block(
chunk: "ModelResponseStream",
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
*,
keep_role: bool,
) -> "ModelResponseStream":
piece: Final = copy.deepcopy(chunk)
role: Final = getattr(piece.choices[0].delta, "role", None) if keep_role else None
thinking: Final = block.get("thinking") if block["type"] == "thinking" else None
piece.choices[0].delta = Delta(role=role, reasoning_content=thinking or None, thinking_blocks=[block])
return piece
@staticmethod
def _split_thinking_blocks(chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
"""Anthropic streams one content block per thinking block, so a delta carrying several is split per block."""
choices: Final = _optional_attr_sequence(chunk, "choices")
if len(choices) != 1:
return (chunk,)
delta: Final = _optional_attr(choices[0], "delta")
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
if len(blocks) < 2 or _optional_attr(delta, "content") or _optional_attr(delta, "tool_calls"):
return (chunk,)
return tuple(
_CombinedChunkSplitter._with_thinking_block(
chunk, _THINKING_BLOCK.validate_python(block), keep_role=position == 0
)
for position, block in enumerate(blocks)
)
def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]":
split_chunks: Final = tuple(
split_chunk
for combined_chunk in self._split(chunk)
for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits
)
return tuple(itertools.chain.from_iterable(self._route_tool_calls(split_chunk) for split_chunk in split_chunks))
block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks)))
return tuple(itertools.chain.from_iterable(map(self._route_tool_calls, block_chunks)))
def __iter__(self) -> "Iterator[ModelResponseStream]":
return self
@ -458,7 +500,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
sent_first_chunk: bool = False
sent_content_block_start: bool = False
sent_content_block_finish: bool = False
current_content_block_type: Literal["text", "tool_use", "thinking"] = "text"
current_content_block_type: Literal["text", "tool_use", "thinking", "redacted_thinking"] = "text"
current_thinking_block_index: int | None = None
sent_last_message: bool = False
holding_chunk: ContentBlockDelta | None = None
holding_stop_reason_chunk: MessageBlockDelta | None = None
@ -1254,13 +1297,14 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
Also gates re-emission after a block transition: when an upstream
chunk both *triggers* a new content block (its type differs from the
active block) and *carries* delta content, that content belongs to
the new block. The synthesized ``content_block_start`` only ever
carries an empty body — see
``_translate_streaming_openai_chunk_to_anthropic_content_block``,
which returns an empty ``TextBlock``/``ToolUseBlock``/thinking block —
so the trigger chunk's delta must be re-queued or the first token of
the new block (the first non-empty text/thinking delta, or bundled
tool arguments) is silently dropped.
the new block. The synthesized ``content_block_start`` from
``_translate_streaming_openai_chunk_to_anthropic_content_block``
carries an empty text, tool-use or unsigned thinking body, and only a
signed thinking block's text rides on the start because its delta is
a ``signature_delta``. So the trigger chunk's delta must be re-queued
or the first token of the new block (the first non-empty text or
unsigned thinking delta, or bundled tool arguments) is silently
dropped.
Delta types outside ``StreamingContentBlockDeltaType`` — the closed
set the translate layer can produce — are treated as empty. The
@ -1352,11 +1396,24 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name)
tool_block["name"] = original_name
thinking_block_index: Final = _first_thinking_block_index(chunk)
previous_thinking_block_index: Final = self.current_thinking_block_index
if thinking_block_index is not None or block_type != self.current_content_block_type:
self.current_thinking_block_index = thinking_block_index
if block_type != self.current_content_block_type:
self.current_content_block_type = block_type
self.current_content_block_start = content_block_start
return True
if (
thinking_block_index is not None
and previous_thinking_block_index is not None
and thinking_block_index != previous_thinking_block_index
):
self.current_content_block_start = content_block_start
return True
# For parallel tool calls, we'll necessarily have a new content block
# if we get a function name since it signals a new tool call
if block_type == "tool_use":

8
litellm/llms/anthropic/pass_through/adapters/transformation.py
View file

@ -1627,7 +1627,7 @@ class LiteLLMAnthropicMessagesAdapter:
def translate_streaming_openai_chunk_to_anthropic_content_block(
self, choices: list[OpenAIStreamingChoice | StreamingChoices]
) -> tuple[
Literal["text", "tool_use", "thinking"],
Literal["text", "tool_use", "thinking", "redacted_thinking"],
"ContentBlockContentBlockDict",
]:
from litellm._uuid import uuid
@ -1664,6 +1664,10 @@ class LiteLLMAnthropicMessagesAdapter:
thinking_blocks = choice.delta.thinking_blocks or []
if len(thinking_blocks) > 0:
thinking_block = thinking_blocks[0]
if thinking_block["type"] == "redacted_thinking":
return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
type="redacted_thinking", data=thinking_block.get("data") or ""
)
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""
signature = thinking_block.get("signature") or ""
@ -1672,7 +1676,7 @@ class LiteLLMAnthropicMessagesAdapter:
assert isinstance(signature, str)
return "thinking", ChatCompletionThinkingBlock(
type="thinking", thinking=thinking, signature=signature
type="thinking", thinking=thinking if signature else "", signature=signature
)
# OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning
# parsers) populate ``reasoning_content`` without ``thinking_blocks``.

2
litellm/llms/anthropic/pass_through/context_management/dispatcher.py
View file

@ -85,6 +85,7 @@ async def apply_context_management(
user_api_key_auth: "UserAPIKeyAuth | None" = None,
sub_call_logging_kwargs: "ProxySubCallLoggingKwargs | None" = None,
parent_trace_id: str | None = None,
count_only: bool = False,
) -> PolyfillResult:
"""Run edits in order; return a single ``PolyfillResult``.
@ -128,6 +129,7 @@ async def apply_context_management(
user_api_key_auth=user_api_key_auth,
sub_call_logging_kwargs=sub_call_logging_kwargs,
parent_trace_id=parent_trace_id,
count_only=count_only,
)
if editor_is_async
else await asyncify(editor)(

3
litellm/llms/anthropic/pass_through/context_management/editors/compact.py
View file

@ -642,6 +642,7 @@ async def apply_compact_20260112(
user_api_key_auth: Optional["UserAPIKeyAuth"] = None,
sub_call_logging_kwargs: ProxySubCallLoggingKwargs | None = None,
parent_trace_id: str | None = None,
count_only: bool = False,
) -> PolyfillResult:
"""Apply ``compact_20260112``; return a ``PolyfillResult``.
@ -682,7 +683,7 @@ async def apply_compact_20260112(
# Opt-in gate: no summary model configured → no-op (but still return the
# Phase A-sliced/stripped messages so compaction blocks don't leak).
summary_model: Final = _read_summary_model_setting()
summary_model: Final = None if count_only else _read_summary_model_setting()
if summary_model is None:
applied["error"] = "summary_model_not_configured"
# Slice-only forwarding: ``augmented_system`` already carries any prior

17
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py
View file

@ -2,6 +2,7 @@ import asyncio
import json
import re
from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence
from contextvars import ContextVar
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
@ -352,6 +353,22 @@ async def aclose_if_supported(stream: object) -> None:
await stream.aclose()
_CLOSING_SUPERSEDED_ATTEMPT: Final[ContextVar[bool]] = ContextVar("closing_superseded_attempt", default=False)
async def aclose_superseded_attempt(stream: object) -> None:
"""Close an attempt a retry or fallback is taking over, so its end-of-stream logging records a failure."""
token: Final = _CLOSING_SUPERSEDED_ATTEMPT.set(True)
try:
await aclose_if_supported(stream)
finally:
_CLOSING_SUPERSEDED_ATTEMPT.reset(token)
def closing_superseded_attempt() -> bool:
return _CLOSING_SUPERSEDED_ATTEMPT.get()
_RESPONSE_HEADERS_ADAPTER: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])

10
litellm/llms/anthropic/pass_through/responses_adapters/handler.py
View file

@ -43,6 +43,12 @@ _BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
logging_obj.record_bridged_responses_api_response(result)
def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, object]:
"""The litellm-specific kwargs forwarded verbatim onto the Responses API request."""
return extra_kwargs or {}
@ -301,7 +307,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
_record_bridged_response(responses_kwargs, result)
return served
@staticmethod
def anthropic_messages_handler(

1
litellm/llms/azure/chat/gpt_5_transformation.py
View file

@ -95,7 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
status_code=400,
message=(
"Azure OpenAI does not support reasoning_effort='none' for this model. "
"Supported values are: 'low', 'medium', and 'high'. "
"To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
"`litellm_settings:\n drop_params: true`\n"
"Issue: https://github.com/BerriAI/litellm/issues/16704"

10
litellm/llms/azure/common_utils.py
View file

@ -2,6 +2,7 @@ import asyncio
import hashlib
import json
import os
import re
from collections.abc import Callable, Mapping
from functools import lru_cache
from types import MappingProxyType
@ -25,6 +26,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.utils import add_path_to_api_base
azure_ad_cache: Final = DualCache()
_LAST_OPENAI_PATH_SEGMENT: Final = re.compile(r"^(.*/openai)(?=/|$)")
class _AzureAdTokenJson(TypedDict, total=False):
@ -798,6 +800,10 @@ class BaseAzureLLM(BaseOpenAILLM):
) -> dict[str, str]: # mutable-ok: mirrors override contract
return cls._base_validate_azure_environment(headers, litellm_params)
@staticmethod
def resolve_azure_api_base(api_base: str | None) -> str | None:
return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
@staticmethod
def _get_base_azure_url(
api_base: str | None,
@ -815,7 +821,7 @@ class BaseAzureLLM(BaseOpenAILLM):
default_api_version: The default API version to use if no api_version is provided. If 'latest', it will use `openai/v1/...` route.
"""
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
api_base = BaseAzureLLM.resolve_azure_api_base(api_base)
if api_base is None:
raise ValueError(
f"api_base is required for Azure AI Studio. Please set the api_base parameter. Passed `api_base={api_base}`"
@ -843,7 +849,7 @@ class BaseAzureLLM(BaseOpenAILLM):
# ensure the request go to /openai/v1 and not just /openai
if "/openai/v1" not in new_url:
parsed_url: Final = httpx.URL(new_url)
new_url = str(parsed_url.copy_with(path=parsed_url.path.replace("/openai", "/openai/v1")))
new_url = str(parsed_url.copy_with(path=_LAST_OPENAI_PATH_SEGMENT.sub(r"\1/v1", parsed_url.path, 1)))
# Use the new query_params dictionary
final_url: Final = httpx.URL(new_url).copy_with(params=query_params)

1
litellm/llms/azure/containers/transformation.py
View file

@ -28,6 +28,7 @@ class AzureContainerConfig(OpenAIContainerConfig):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return BaseAzureLLM.base_validate_azure_environment(
headers=headers,

12
litellm/llms/azure/fine_tuning/handler.py
View file

@ -1,4 +1,4 @@
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from typing import Final, cast
import httpx
@ -66,6 +66,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
self._ensure_training_type(create_fine_tuning_job_data)
@ -78,6 +79,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -109,6 +111,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -119,6 +122,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -149,6 +153,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
max_retries: int | None,
organization: str | None,
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> LiteLLMFineTuningJob | Coroutine[object, object, LiteLLMFineTuningJob]:
openai_client: Final[OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None] = self.get_openai_client(
api_key=api_key,
@ -159,6 +164,7 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client=client,
_is_async=_is_async,
api_version=api_version,
litellm_params=litellm_params,
)
if openai_client is None:
raise ValueError(
@ -188,14 +194,14 @@ class AzureOpenAIFineTuningAPI(OpenAIFineTuningAPI, BaseAzureLLM):
client: OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None = None,
_is_async: bool = False,
api_version: str | None = None,
litellm_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> OpenAI | AsyncOpenAI | AzureOpenAI | AsyncAzureOpenAI | None:
# Override to use Azure-specific client initialization
if isinstance(client, OpenAI) or isinstance(client, AsyncOpenAI):
client = None
return self.get_azure_openai_client(
litellm_params=litellm_params or {},
litellm_params=dict(litellm_params) if litellm_params is not None else None,
api_key=api_key,
api_base=api_base,
api_version=api_version,

39
litellm/llms/azure/responses/transformation.py
View file

@ -1,4 +1,7 @@
import re
from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
import httpx
@ -6,6 +9,7 @@ from openai.types.responses import ResponseReasoningItem
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -25,6 +29,34 @@ else:
_DECODED_JSON: Final = TypeAdapter(object)
_DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
value: Final = litellm_params.get(key)
return value if isinstance(value, str) and value else None
def _resolve_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> str | None:
"""
Dated GA api_versions (e.g. 2024-10-21) predate the Responses API on Azure and 404 there, so a deployment
pinned to one for chat gets the default Responses version unless it sets responses_api_version.
"""
api_version: Final = _str_param(litellm_params, "api_version")
if api_base is not None and "api-version" in httpx.URL(api_base).params:
return api_version
responses_api_version: Final = _str_param(litellm_params, "responses_api_version")
if responses_api_version is not None:
return responses_api_version
if api_version is not None and _DATED_GA_API_VERSION.fullmatch(api_version):
return AZURE_DEFAULT_RESPONSES_API_VERSION
return api_version
def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Parameters not supported by Azure Responses API
AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@ -170,11 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
- A complete URL string, e.g.,
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
return BaseAzureLLM.get_base_azure_url(
api_base=api_base,
litellm_params=litellm_params,
api_base=resolved_api_base,
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)

3
litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py
View file

@ -13,7 +13,7 @@ from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
AzureAIAnthropicCountTokensHandler,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
# Global handler instance - reuse across all token counting requests
azure_ai_anthropic_count_tokens_handler: Final = AzureAIAnthropicCountTokensHandler()
@ -54,6 +54,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
system: object | None = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
"""
Count tokens using Azure AI Anthropic's CountTokens API.

29
litellm/llms/azure_ai/chat/transformation.py
View file

@ -1,6 +1,7 @@
import copy
import enum
import re
from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Final, cast
import httpx
@ -21,12 +22,12 @@ from litellm.llms.azure_ai.common_utils import (
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
from litellm.llms.openai.openai import OpenAIConfig
from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig
from litellm.llms.xai.chat.transformation import XAIChatConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelResponse, ProviderField
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
from litellm.utils import add_path_to_api_base, supports_tool_choice
if TYPE_CHECKING:
@ -45,6 +46,14 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
)
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
usage: Final = chunk.get("usage")
if isinstance(usage, dict):
XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
return super().chunk_parser(chunk)
class AzureAIGPT5Config(OpenAIGPT5Config):
@classmethod
def _model_map_lookup_name(cls, model: str) -> str:
@ -319,7 +328,7 @@ class AzureAIStudioConfig(OpenAIConfig):
json_mode: bool | None = None,
) -> ModelResponse:
model_response.model = f"azure_ai/{model}"
return super().transform_response(
response: Final = super().transform_response(
model=model,
raw_response=raw_response,
model_response=model_response,
@ -332,6 +341,20 @@ class AzureAIStudioConfig(OpenAIConfig):
api_key=api_key,
json_mode=json_mode,
)
XAIChatConfig.fold_reasoning_tokens_into_completion(response)
return response
def get_model_response_iterator(
self,
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
sync_stream: bool,
json_mode: bool | None = False,
) -> "AzureAIChatCompletionResponseIterator":
return AzureAIChatCompletionResponseIterator(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
def should_retry_llm_api_inside_llm_translation_on_http_error(
self, e: httpx.HTTPStatusError, litellm_params: dict

69
litellm/llms/azure_ai/decisions/transformation.py
View file

@ -1,11 +1,21 @@
import asyncio
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import ValidationError
import litellm
from litellm.llms.azure_ai.common_utils import (
AZURE_ENTRA_LITELLM_PARAM_KEYS,
get_azure_ai_auth_headers,
)
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
_FOUNDRY_ROUTE_SUFFIX: Final = re.compile(r"(/api/projects/[^/]+)?(/openai/v1|/openai|/models|/v1)?$")
_AZURE_AI_CREDENTIAL_PARAM_KEYS: Final = AZURE_ENTRA_LITELLM_PARAM_KEYS | frozenset({"azure_ad_token_provider"})
class AzureAIDecisionsConfig(BaseDecisionsConfig):
@ -14,10 +24,69 @@ class AzureAIDecisionsConfig(BaseDecisionsConfig):
path = "/providers/microsoft/v1/systemone"
api_key_env = ("AZURE_AI_API_KEY",)
api_base_env = ("AZURE_AI_API_BASE",)
api_key_required = False
def missing_api_base_message(self, custom_llm_provider: str) -> str:
return "Missing AZURE_AI_API_BASE - set AZURE_AI_API_BASE or pass api_base, e.g. https://<resource>.services.ai.azure.com"
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
credential_params: Final = MappingProxyType(
{key: litellm_params[key] for key in _AZURE_AI_CREDENTIAL_PARAM_KEYS if key in litellm_params}
)
try:
auth_headers: Final = get_azure_ai_auth_headers(api_key, credential_params)
except ValidationError as error:
field_names: Final = tuple(
dict.fromkeys(
".".join(str(field) for field in detail["loc"])
for detail in error.errors(include_input=False, include_context=False, include_url=False)
)
)
raise litellm.BadRequestError(
message=f"Invalid Azure AI credential parameter(s): {', '.join(f'`{name}`' for name in field_names)}",
model=model,
llm_provider="azure_ai",
) from error
except ValueError as error:
raise litellm.AuthenticationError(message=str(error), model=model, llm_provider="azure_ai") from error
return {
**super().validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
),
**auth_headers,
}
async def avalidate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
if api_key:
return self.validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
return await asyncio.to_thread(
self.validate_environment,
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
def get_complete_url(self, api_base: str, model: str) -> str:
url: Final = httpx.URL(api_base)
resource_path: Final = _FOUNDRY_ROUTE_SUFFIX.sub("", url.path.rstrip("/"))

26
litellm/llms/base_llm/base_model_iterator.py
View file

@ -6,16 +6,38 @@ import litellm
if TYPE_CHECKING:
import httpx
from litellm.types.llms.openai import ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock
from litellm.types.utils import (
Choices,
Delta,
GenericStreamingChunk,
Message,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
def _indexed_thinking_block(
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, index: int
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock:
if block["type"] == "redacted_thinking":
return {**block, "index": index}
return {**block, "index": index}
def _streaming_delta(message: Message) -> Delta:
thinking_blocks: Final = message.thinking_blocks if hasattr(message, "thinking_blocks") else None
return Delta(
**message.model_dump(exclude={"thinking_blocks"}),
thinking_blocks=(
None
if thinking_blocks is None
else [_indexed_thinking_block(block, index) for index, block in enumerate(thinking_blocks)]
),
)
def convert_model_response_to_streaming(
model_response: ModelResponse,
) -> ModelResponseStream:
@ -40,9 +62,7 @@ def convert_model_response_to_streaming(
streaming_choices.append(
StreamingChoices(
index=choice.index,
delta=Delta(
**cast(Choices, choice).message.model_dump(),
),
delta=_streaming_delta(cast(Choices, choice).message),
finish_reason=choice.finish_reason,
)
)

9
litellm/llms/base_llm/base_utils.py
View file

@ -16,7 +16,13 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import (
CountContextManagement,
Message,
ProviderSpecificModelInfo,
TokenCountMessagesFormat,
TokenCountResponse,
)
class BaseTokenCounter(ABC):
@ -32,6 +38,7 @@ class BaseTokenCounter(ABC):
system: Any | None = None,
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
pass

16
litellm/llms/base_llm/chat/transformation.py
View file

@ -8,7 +8,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final, Union
import httpx
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
from litellm.constants import DEFAULT_MAX_TOKENS, RESPONSE_FORMAT_TOOL_NAME
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -38,6 +38,9 @@ else:
LiteLLMLoggingObj = Any
REQUEST_PARAMS: Final = TypeAdapter(Mapping[str, object])
class BaseLLMException(Exception):
def __init__(
self,
@ -126,6 +129,17 @@ class BaseConfig(ABC):
thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None
return thinking is True or thinking_type == "enabled" or non_default_params.get("reasoning_effort") is not None
def is_thinking_requested(self, non_default_params: Mapping[str, object]) -> bool:
match non_default_params:
case {"thinking": True | {"type": "enabled" | "adaptive"}}:
return True
case {"reasoning_effort": None | "none"}:
return False
case {"reasoning_effort": object()}:
return True
case _:
return False
def is_max_tokens_in_request(self, non_default_params: dict) -> bool:
"""
OpenAI spec allows max_tokens or max_completion_tokens to be specified.

1
litellm/llms/base_llm/containers/transformation.py
View file

@ -81,6 +81,7 @@ class BaseContainerConfig(ABC):
self,
headers: dict,
api_key: str | None = None,
litellm_params: GenericLiteLLMParams | None = None,
) -> dict:
return {}

45
litellm/llms/base_llm/decisions/transformation.py
View file

@ -390,8 +390,24 @@ def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRe
)
_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...])
def _normalize_answer(value: object) -> Mapping[str, object]:
answer: Final = _MAPPING.validate_python(value)
legend: Final = answer.get("legend")
if answer.get("type") != "score" or not isinstance(legend, list):
return answer
labels: Final = _LABELS.validate_python(legend)
return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}}
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request)
response: Final = _MAPPING.validate_python(payload)
answers: Final = _MAPPING.validate_python(response.get("answers"))
normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}}
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request)
def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
@ -495,6 +511,11 @@ class BaseDecisionsConfig(ABC):
def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]:
return self.resolve_api_base(api_base), self.resolve_api_key(api_key)
def resolve_credential(
self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
) -> str | None:
return api_key
@staticmethod
def _first_secret(names: tuple[str, ...]) -> str | None:
return next((value for value in (get_secret_str(name) for name in names) if value), None)
@ -505,13 +526,33 @@ class BaseDecisionsConfig(ABC):
def request_model(self, model: str) -> str:
return model
def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]:
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
return {
**{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS},
**({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}),
"Content-Type": "application/json",
}
async def avalidate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
return self.validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
def get_complete_url(self, api_base: str, model: str) -> str:
return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}"

5
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -203,6 +203,11 @@ class BaseTranslation(ABC):
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
return None
def needs_ended_stream_scan(self, guardrail: "CustomGuardrail", responses_so_far: Sequence[object]) -> bool:
"""Whether the end-of-stream round scans ``guardrail`` even when a sampled round already covered every
text, because this translation hands that guardrail marks only the ended round carries."""
return False
def released_stream_as_ended(self, responses_so_far: Sequence[object]) -> tuple[object, ...]:
"""The chunks a client left the stream with, closed the way this endpoint ends a stream, so the
end-of-stream scan also inspects tool calls the stream never finished"""

13
litellm/llms/base_llm/guardrail_translation/utils.py
View file

@ -4,7 +4,7 @@ import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Final, TypeVar, cast # noqa: TID251 # a rebuilt chat row has no typed constructor across roles
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.openai import AllMessageValues, ResponseAPIUsage
@ -292,6 +292,17 @@ def openai_tool_name(tool: object) -> str | None:
return flat_name if isinstance(flat_name, str) else None
_FIELDS: Final = TypeAdapter(Mapping[object, object], config=ConfigDict(strict=True))
_ITEMS: Final = TypeAdapter(Sequence[object], config=ConfigDict(strict=True))
def list_field(data: object, key: str) -> Sequence[object]:
try:
return _ITEMS.validate_python(_FIELDS.validate_python(data).get(key))
except ValidationError:
return ()
def anthropic_tool_names(tool: object) -> tuple[str, ...]:
"""Every name a /v1/messages tool dict can act under: the flat Anthropic ``name`` plus
``function.name`` for OpenAI-format tools the bridge forwards verbatim. Allowlist checks

10
litellm/llms/base_llm/realtime/transcription_protocol.py
View file

@ -102,6 +102,16 @@ def json_integer(
return value
def uses_server_vad(
update: TranscriptionSessionUpdate, backend: str, drop_params: bool, error: ProtocolErrorType
) -> bool:
if update.turn_detection_disabled:
return False
if update.turn_detection_type in (None, "server_vad") or drop_params:
return True
raise error(f"{backend} supports server_vad turn detection or null. Set drop_params to use server_vad instead")
def new_event_id() -> str:
return f"event_{uuid.uuid4().hex}"

123
litellm/llms/base_llm/videos/async_task_utils.py Normal file
View file

@ -0,0 +1,123 @@
import base64
from collections.abc import Mapping
from io import BufferedReader, BytesIO
from itertools import accumulate, repeat
from pathlib import Path, PurePath
from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.images.utils import ImageEditRequestUtils
from litellm.types.videos.utils import decode_video_id_with_provider
_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
def _fields(value: object) -> Mapping[str, object] | None:
try:
return _FIELDS_ADAPTER.validate_python(value)
except ValidationError:
return None
def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
return _fields(getattr(logging_obj, "litellm_params", None)) or {}
def _id_safe_name(name: object) -> str | None:
return name if isinstance(name, str) and name and ";" not in name else None
def _stamped_models(metadata: object) -> tuple[str, str | None] | None:
fields: Final = _fields(metadata)
if fields is None:
return None
requested: Final = _id_safe_name(fields.get("model_group"))
served: Final = _id_safe_name(fields.get("deployment_model_name"))
if requested is None:
return None
return requested, None if served is None or served == requested or "*" in served else served
def routed_video_models(logging_obj: object) -> tuple[str, str | None] | None:
"""
A later status or content request checks the key's model access against the model encoded in the id, and
routes by it. One upstream model name can be served by several deployments (regions, accounts), so only the
group that served the task can send its polls back to the deployment that created it, while a key may be
scoped to the alias, routing group or wildcard name it asked for instead. The id carries both names: the one
the router was asked for on this hop, and the group that served it when that differs and is not a wildcard.
The router stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
"""
params: Final = _logged_litellm_params(logging_obj)
stamped: Final = (_stamped_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
return next((models for models in stamped if models), None)
def polled_video_models(logging_obj: object) -> tuple[str | None, str | None]:
"""
The served group survives into the returned id only when the router sent this poll to it, so an edited id
cannot write any name it likes back.
"""
video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
if not isinstance(video_id, str):
return None, None
decoded: Final = decode_video_id_with_provider(video_id)
served: Final = decoded.get("served_model")
routed: Final = routed_video_models(logging_obj)
return decoded.get("model_id") or None, served if routed is None or served in routed else None
def _tuple_payload(file_obj: object) -> object:
if not isinstance(file_obj, tuple):
return file_obj
file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
return file_tuple[1] if len(file_tuple) >= 2 else None
def _file_content(file_obj: object) -> object:
unwrapped: Final = accumulate(repeat(None), lambda current, _: _tuple_payload(current), initial=file_obj)
return next(item for item in unwrapped if not isinstance(item, tuple))
def input_reference_bytes(file_obj: object) -> bytes:
content: Final = _file_content(file_obj)
if isinstance(content, PurePath):
return Path(content).read_bytes()
if isinstance(content, (BytesIO, BufferedReader)):
current_position: Final = content.tell()
_ = content.seek(0)
data: Final = content.read()
_ = content.seek(current_position)
return data
if isinstance(content, bytes):
return content
if isinstance(content, bytearray):
return bytes(content)
read: Final = getattr(content, "read", None)
if callable(read):
read_data: Final[object] = read()
if isinstance(read_data, bytes):
return read_data
raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
def _reference_object_url(reference: Mapping[str, object]) -> str:
image_url: Final = reference.get("image_url")
if isinstance(image_url, str) and image_url:
return image_url
raise ValueError("input_reference as an object must carry an image_url, a file_id is not supported")
def input_reference_url(image: object) -> str:
"""A string is a URL the provider fetches itself and is never opened as a local path. OpenAI's object form
carries that URL, or a data URL, in ``image_url``."""
if isinstance(image, str):
return image
reference: Final = _fields(image)
if reference is not None:
return _reference_object_url(reference)
content: Final = input_reference_bytes(image)
content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
encoded: Final = base64.b64encode(content).decode("utf-8")
return f"data:{content_type};base64,{encoded}"

19
litellm/llms/bedrock/chat/converse_handler.py
View file

@ -1,5 +1,5 @@
import json
from collections.abc import Iterator
from collections.abc import Iterator, Mapping
from typing import Final
import httpx
@ -10,6 +10,7 @@ from litellm.anthropic_beta_headers_manager import (
)
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.base_llm import base_model_iterator
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -21,7 +22,7 @@ from litellm.utils import CustomStreamWrapper
from ..base_aws_llm import BaseAWSLLM, Credentials, bedrock_bearer_token, pop_aws_auth_params, run_aws_signing
from ..common_utils import BedrockError, error_response_text, get_all_bedrock_regions
from .invoke_handler import AWSEventStreamDecoder, MockResponseIterator, make_call
from .invoke_handler import AWSEventStreamDecoder, make_call
def make_sync_call(
@ -36,7 +37,8 @@ def make_sync_call(
fake_stream: bool = False,
stream_chunk_size: int | None = None,
timeout: float | httpx.Timeout | None = None,
) -> tuple[MockResponseIterator | Iterator[object], httpx.Headers]:
optional_params: Mapping[str, object] | None = None,
) -> tuple[base_model_iterator.MockResponseIterator | Iterator[object], httpx.Headers]:
if client is None:
client = get_httpx_client() # Create a new client if none provided
@ -64,14 +66,15 @@ def make_sync_call(
model_response=litellm.ModelResponse(),
stream=True,
logging_obj=logging_obj,
optional_params={},
optional_params=dict(optional_params or {}),
api_key="",
data=data,
messages=messages,
encoding=litellm.encoding,
json_mode=json_mode,
)
completion_stream: MockResponseIterator | Iterator[object] = MockResponseIterator(
model_response=model_response, json_mode=json_mode
completion_stream: base_model_iterator.MockResponseIterator | Iterator[object] = (
base_model_iterator.MockResponseIterator(model_response=model_response)
)
else:
decoder: Final = AWSEventStreamDecoder(model=model, json_mode=json_mode)
@ -158,6 +161,7 @@ class BedrockConverseLLM(BaseAWSLLM):
json_mode=json_mode,
stream_chunk_size=stream_chunk_size,
timeout=timeout,
optional_params=optional_params,
)
streaming_response: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -290,7 +294,7 @@ class BedrockConverseLLM(BaseAWSLLM):
stream_chunk_size: Final = stored_control_options(litellm_params).stream_chunk_size if stream is True else None
unencoded_model_id: Final = optional_params.pop("model_id", None)
fake_stream = optional_params.pop("fake_stream", False)
json_mode: Final = optional_params.get("json_mode", False)
json_mode: Final = optional_params.get("json_mode") is True
if unencoded_model_id is not None:
modelId = self.encode_model_id(model_id=unencoded_model_id)
else:
@ -463,6 +467,7 @@ class BedrockConverseLLM(BaseAWSLLM):
fake_stream=fake_stream,
stream_chunk_size=stream_chunk_size,
timeout=timeout,
optional_params=optional_params,
)
streaming_response: Final = CustomStreamWrapper(
completion_stream=completion_stream,

111
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -29,6 +29,7 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
drop_lookaround_regex_patterns,
is_object_schema,
native_redacted_thinking_data,
parse_content_for_reasoning,
tool_with_sanitized_parameters,
@ -53,9 +54,16 @@ from litellm.llms.anthropic.chat.transformation import (
tools_without_unsupported_tool_search,
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.base_llm.chat.transformation import REQUEST_PARAMS, BaseConfig, BaseLLMException
from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_images
from litellm.llms.bedrock.common_utils import bedrock_model_supports_regex_lookaround, bedrock_supports_tool_search
from litellm.llms.bedrock.common_utils import (
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_model_supports_regex_lookaround,
bedrock_native_output_format,
bedrock_supports_tool_search,
deployment_model_info,
)
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
bedrock_request_metadata_is_owned,
@ -939,17 +947,22 @@ class AmazonConverseConfig(BaseConfig):
return _tool
@staticmethod
def _supports_native_structured_outputs(model: str, custom_llm_provider: str | None = None) -> bool:
"""Check if the Bedrock model supports native structured outputs (outputConfig.textFormat).
def _supports_native_structured_outputs(
model: str,
custom_llm_provider: str | None,
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> bool:
"""Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
Delegates to the standard ``supports_native_structured_output`` utility
which looks up the flag in ``litellm.model_cost`` via
``_get_model_info_helper``.
``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) only when the
request asks for thinking, which the json tool path would drop.
Ref: https://docs.aws.amazon.com/bedrock/latest/userguide/structured-output.html
"""
from litellm.utils import supports_native_structured_output
return supports_native_structured_output(model=model, custom_llm_provider=custom_llm_provider)
return bedrock_chat_uses_native_structured_output(
model, thinking_requested, custom_llm_provider or "bedrock_converse", model_info
)
@staticmethod
def _add_additional_properties_to_schema(schema: dict) -> dict:
@ -964,9 +977,9 @@ class AmazonConverseConfig(BaseConfig):
if not isinstance(schema, dict):
return schema
result: Final = dict(schema)
result: Final[dict[str, object]] = dict(schema)
if result.get("type") == "object" and "additionalProperties" not in result:
if is_object_schema(result) and "additionalProperties" not in result:
result["additionalProperties"] = False
# Recurse into nested schemas
@ -977,16 +990,46 @@ class AmazonConverseConfig(BaseConfig):
if "items" in result and isinstance(result["items"], dict):
result["items"] = AmazonConverseConfig._add_additional_properties_to_schema(result["items"])
for defs_key in ("$defs", "definitions"):
if defs_key in result and isinstance(result[defs_key], dict):
if isinstance(definitions := result.get(defs_key), dict):
result[defs_key] = {
k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in result[defs_key].items()
k: AmazonConverseConfig._add_additional_properties_to_schema(v) for k, v in definitions.items()
}
for key in ("anyOf", "allOf", "oneOf"):
if key in result and isinstance(result[key], list):
result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in result[key]]
if isinstance(variants := result.get(key), list):
result[key] = [AmazonConverseConfig._add_additional_properties_to_schema(item) for item in variants]
return result
def _native_output_schema(
self,
model: str,
json_schema: Mapping[str, object],
thinking_requested: bool,
model_info: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
if not self._supports_native_structured_outputs(
model, self.custom_llm_provider, thinking_requested, model_info
):
return None
if BedrockModelInfo.get_base_model(model).startswith("anthropic"):
return bedrock_cleaned_output_schema(model, json_schema, self.custom_llm_provider or "bedrock_converse")
return json_schema
def _output_config_for_anthropic_format(
self, model: str, output_config_format: Mapping[str, object], model_info: Mapping[str, object] | None
) -> OutputConfigBlock | None:
native_format: Final = bedrock_native_output_format(
model, output_config_format, self.custom_llm_provider or "bedrock_converse", model_info
)
if native_format is None:
return None
schema, name, description = (native_format.get(key) for key in ("schema", "name", "description"))
return AmazonConverseConfig._create_output_config_for_response_format(
json_schema={**schema} if isinstance(schema, Mapping) else None,
name=name if isinstance(name, str) else None,
description=description if isinstance(description, str) else None,
)
@staticmethod
def _create_output_config_for_response_format(
json_schema: dict | None = None,
@ -1046,6 +1089,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
model: str,
drop_params: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
is_thinking_enabled: Final = self.is_thinking_enabled(non_default_params)
base_model: Final = BedrockModelInfo.get_base_model(model)
@ -1062,6 +1106,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params=optional_params,
non_default_params=non_default_params,
is_thinking_enabled=is_thinking_enabled,
model_info=model_info,
)
if param == "max_tokens" or param == "max_completion_tokens":
optional_params["maxTokens"] = (
@ -1273,6 +1318,7 @@ class AmazonConverseConfig(BaseConfig):
optional_params: dict,
non_default_params: dict,
is_thinking_enabled: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
"""
Handles translation of response_format parameter to Bedrock format.
@ -1296,13 +1342,23 @@ class AmazonConverseConfig(BaseConfig):
if "type" in value and value["type"] == "text":
return optional_params
if self._supports_native_structured_outputs(model, self.custom_llm_provider) and json_schema is not None:
native_schema: Final = (
self._native_output_schema(
model,
json_schema,
self.is_thinking_requested(REQUEST_PARAMS.validate_python(non_default_params)),
model_info,
)
if json_schema is not None
else None
)
if native_schema is not None:
# Use Bedrock's native structured outputs API (outputConfig.textFormat)
# No synthetic tool injection, no fake_stream needed.
# Requires an explicit schema — json_object with no schema falls through
# to the tool-call path below.
output_config: Final = self._create_output_config_for_response_format(
json_schema=json_schema,
json_schema=dict(native_schema),
name=name,
description=description,
)
@ -1644,7 +1700,11 @@ class AmazonConverseConfig(BaseConfig):
tool_choice.pop("type", None)
def _prepare_request_params(
self, optional_params: dict, model: str, drop_params: bool = False
self,
optional_params: dict,
model: str,
drop_params: bool = False,
model_info: Mapping[str, object] | None = None,
) -> tuple[dict, dict, dict, OutputConfigBlock | None]:
"""Prepare and separate request parameters."""
# Consume the internal ``_output_config_normalized`` marker set by
@ -1687,17 +1747,12 @@ class AmazonConverseConfig(BaseConfig):
and output_config_format is not None
and output_config_format.get("type") == "json_schema"
and base_model.startswith("anthropic")
and self._supports_native_structured_outputs(model, self.custom_llm_provider)
):
output_config = self._create_output_config_for_response_format(
json_schema=output_config_format.get("schema"),
name=output_config_format.get("name"),
description=output_config_format.get("description"),
)
elif output_config is None and output_config_format is not None:
output_config = self._output_config_for_anthropic_format(model, output_config_format, model_info)
if output_config is None and output_config_format is not None:
litellm.verbose_logger.warning(
"Bedrock Converse: dropping `output_config.format` for model=%s — "
"model does not advertise `supports_native_structured_output` in "
"model does not take native structured output for this schema per "
"model_prices_and_context_window.json. The schema will not be "
"enforced; pass `response_format` to use the synthetic tool-call "
"fallback.",
@ -1987,7 +2042,7 @@ class AmazonConverseConfig(BaseConfig):
additional_request_params,
request_metadata,
output_config,
) = self._prepare_request_params(optional_params, model, drop_params)
) = self._prepare_request_params(optional_params, model, drop_params, deployment_model_info(litellm_params))
original_tools: Final = inference_params.pop("tools", [])

125
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -1,7 +1,7 @@
import json
import types
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import TYPE_CHECKING, Final, cast
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import TypeAdapter, ValidationError
@ -20,6 +20,7 @@ from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
from litellm.llms.anthropic.chat.transformation import JsonModeRequest
from litellm.llms.base_llm import base_model_iterator
from litellm.llms.base_llm.base_utils import ResponseFormatSchema
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -32,12 +33,8 @@ from litellm.types.llms.openai import (
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionUsageBlock,
)
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
Delta,
ModelResponse,
ModelResponseStream,
@ -228,7 +225,10 @@ async def make_call(
bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
stream_chunk_size: int | None = None,
timeout: float | httpx.Timeout | None = None,
) -> "tuple[MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
optional_params: Mapping[str, object] | None = None,
) -> tuple[
base_model_iterator.MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict], httpx.Headers
]:
try:
if client is None:
client = get_async_httpx_client(
@ -264,15 +264,16 @@ async def make_call(
model_response=litellm.ModelResponse(),
stream=True,
logging_obj=logging_obj,
optional_params={},
optional_params=dict(optional_params or {}),
api_key="",
data=data,
messages=messages,
encoding=litellm.encoding,
json_mode=json_mode,
)
completion_stream: MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict] = (
MockResponseIterator(model_response=model_response, json_mode=json_mode)
)
completion_stream: (
base_model_iterator.MockResponseIterator | AsyncIterator[GChunk | ModelResponseStream | dict]
) = base_model_iterator.MockResponseIterator(model_response=model_response)
else:
decoder: Final = _stream_decoder(
bedrock_invoke_provider,
@ -324,7 +325,7 @@ def make_sync_call(
bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
stream_chunk_size: int | None = None,
timeout: float | httpx.Timeout | None = None,
) -> "tuple[MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]":
) -> tuple[base_model_iterator.MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict], httpx.Headers]:
try:
if client is None:
client = get_httpx_client(
@ -364,10 +365,11 @@ def make_sync_call(
data=data,
messages=messages,
encoding=litellm.encoding,
json_mode=json_mode,
)
completion_stream: MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict] = (
MockResponseIterator(model_response=model_response, json_mode=json_mode)
)
completion_stream: (
base_model_iterator.MockResponseIterator | Iterator[GChunk | ModelResponseStream | dict]
) = base_model_iterator.MockResponseIterator(model_response=model_response)
else:
decoder: Final = _stream_decoder(
bedrock_invoke_provider,
@ -991,98 +993,3 @@ class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)
class MockResponseIterator: # for returning ai21 streaming responses
def __init__(self, model_response, json_mode: bool | None = False):
self.model_response = model_response
self.json_mode = json_mode
self.is_done = False
# Sync iterator
def __iter__(self):
return self
def _handle_json_mode_chunk(
self, text: str, tool_calls: list[ChatCompletionToolCallChunk] | None
) -> tuple[str, ChatCompletionToolCallChunk | None]:
"""
If JSON mode is enabled, convert the tool call to a message.
Bedrock returns the JSON schema as part of the tool call
OpenAI returns the JSON schema as part of the content, this handles placing it in the content
Args:
text: str
tool_use: Optional[ChatCompletionToolCallChunk]
Returns:
Tuple[str, Optional[ChatCompletionToolCallChunk]]
text: The text to use in the content
tool_use: The ChatCompletionToolCallChunk to use in the chunk response
"""
tool_use: ChatCompletionToolCallChunk | None = None
if self.json_mode is True and tool_calls is not None:
message: Final = litellm.AnthropicConfig().convert_tool_response_to_message(tool_calls=tool_calls)
if message is not None:
text = message.content or ""
tool_use = None
elif tool_calls is not None and len(tool_calls) > 0:
tool_use = tool_calls[0]
return text, tool_use
def _chunk_parser(self, chunk_data: ModelResponse) -> GChunk:
try:
chunk_usage: Final[Usage] = getattr(chunk_data, "usage")
text = chunk_data.choices[0].message.content or ""
tool_use = None
_model_response_tool_call: Final = cast(
list[ChatCompletionMessageToolCall] | None,
cast(Choices, chunk_data.choices[0]).message.tool_calls,
)
if self.json_mode is True:
text, tool_use = self._handle_json_mode_chunk(
text=text,
tool_calls=chunk_data.choices[0].message.tool_calls,
)
elif _model_response_tool_call is not None:
tool_use = ChatCompletionToolCallChunk(
id=_model_response_tool_call[0].id,
type="function",
function=ChatCompletionToolCallFunctionChunk(
name=_model_response_tool_call[0].function.name,
arguments=_model_response_tool_call[0].function.arguments,
),
index=0,
)
processed_chunk: Final = GChunk(
text=text,
tool_use=tool_use,
is_finished=True,
finish_reason=map_finish_reason(finish_reason=chunk_data.choices[0].finish_reason or ""),
usage=ChatCompletionUsageBlock(
prompt_tokens=chunk_usage.prompt_tokens,
completion_tokens=chunk_usage.completion_tokens,
total_tokens=chunk_usage.total_tokens,
),
index=0,
)
return processed_chunk
except Exception as e:
raise ValueError(f"Failed to decode chunk: {chunk_data}. Error: {e}")
def __next__(self):
if self.is_done:
raise StopIteration
self.is_done = True
return self._chunk_parser(self.model_response)
# Async iterator
def __aiter__(self):
return self
async def __anext__(self):
if self.is_done:
raise StopAsyncIteration
self.is_done = True
return self._chunk_parser(self.model_response)

5
litellm/llms/bedrock/chat/invoke_transformations/amazon_nova_transformation.py
View file

@ -92,8 +92,11 @@ class AmazonInvokeNovaConfig(AmazonInvokeConfig, AmazonConverseConfig):
optional_params: dict,
model: str,
drop_params: bool,
model_info: Mapping[str, object] | None = None,
) -> dict:
return AmazonConverseConfig.map_openai_params(self, non_default_params, optional_params, model, drop_params)
return AmazonConverseConfig.map_openai_params(
self, non_default_params, optional_params, model, drop_params, model_info
)
def transform_request(
self,

31
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -23,7 +23,10 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
)
from litellm.llms.bedrock.common_utils import (
apply_bedrock_invoke_structured_output,
bedrock_chat_uses_native_structured_output,
bedrock_cleaned_output_schema,
bedrock_supports_tool_search,
deployment_model_info,
get_anthropic_beta_from_headers,
normalize_bedrock_opus_output_config_effort,
normalize_custom_field_on_tools,
@ -32,6 +35,7 @@ from litellm.llms.bedrock.common_utils import (
tools_without_eager_input_streaming,
)
from litellm.types.llms.anthropic import (
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER,
ANTHROPIC_TOOL_SEARCH_BETA_HEADER,
AnthropicThinkingParam,
@ -48,6 +52,7 @@ else:
LiteLLMLoggingObj = Any
_NON_DEFAULT_PARAMS: Final = TypeAdapter(Mapping[str, object])
_LITELLM_PARAMS: Final = TypeAdapter(Mapping[str, object])
class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
@ -70,6 +75,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
anthropic_version: str = "bedrock-2023-05-31"
def __init__(self, model_info: Mapping[str, object] | None = None) -> None:
super().__init__()
self._deployment_model_info: Final = model_info
@property
def custom_llm_provider(self) -> str | None:
return "bedrock"
@ -80,10 +89,18 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
def get_supported_openai_params(self, model: str) -> list[str]:
return AnthropicConfig.get_supported_openai_params(self, model)
def uses_native_structured_output(self, model: str) -> bool:
from litellm.utils import supports_native_structured_output
def native_output_format_schema(self, json_schema: Mapping[str, object]) -> Mapping[str, object] | None:
return AnthropicConfig.native_output_schema(json_schema)
return supports_native_structured_output(model=model, custom_llm_provider="bedrock")
def uses_native_structured_output_for_schema(
self,
model: str,
json_schema: Mapping[str, object] | None,
request_params: Mapping[str, object],
) -> bool:
return bedrock_chat_uses_native_structured_output(
model, self.is_thinking_requested(request_params), model_info=self._deployment_model_info
) and (json_schema is None or bedrock_cleaned_output_schema(model, json_schema) is not None)
def map_openai_params(
self,
@ -239,6 +256,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
apply_bedrock_invoke_structured_output(
model=model,
request_body=anthropic_request,
model_info=deployment_model_info(_LITELLM_PARAMS.validate_python(litellm_params)),
)
strip_unsupported_bedrock_invoke_output_config_keys(
model=model,
@ -270,7 +288,12 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
is_mid_conversation_output_config_used: Final = self.is_mid_conversation_output_config_used(messages)
is_thinking_display_updates_used: Final = self.is_thinking_display_updates_used(thinking)
user_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
client_beta_set: Final = set(get_anthropic_beta_from_headers(headers))
user_beta_set: Final = (
client_beta_set
if "safeguards" in optional_params
else client_beta_set - {ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value}
)
beta_set: Final = set(user_beta_set)
auto_betas: Final = self.get_anthropic_beta_list(
model=model,

104
litellm/llms/bedrock/common_utils.py
View file

@ -29,6 +29,7 @@ import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -249,15 +250,99 @@ def convert_bedrock_invoke_output_format_to_inline_schema(
request_body["messages"] = new_messages
def _bedrock_model_supports(model: str, key: str) -> bool:
def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "bedrock") -> bool:
from litellm.utils import supports_factory
return supports_factory(model=model, custom_llm_provider="bedrock", key=key)
return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key=key)
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
def deployment_model_info(litellm_params: Mapping[str, object] | None) -> Mapping[str, object]:
model_info: Final = (litellm_params or {}).get("model_info")
return _DEPLOYMENT_MODEL_INFO.validate_python(model_info) if isinstance(model_info, Mapping) else {}
def _bedrock_native_output_opt_in(
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None
) -> bool:
deployment_flag: Final = (model_info or {}).get("supports_bedrock_native_structured_output")
if isinstance(deployment_flag, bool):
return deployment_flag
return _bedrock_model_supports(model, "supports_bedrock_native_structured_output", custom_llm_provider)
def bedrock_supports_native_structured_output(
model: str, custom_llm_provider: str = "bedrock", model_info: Mapping[str, object] | None = None
) -> bool:
return _bedrock_model_supports(
model, "supports_native_structured_output", custom_llm_provider
) or _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
def bedrock_chat_uses_native_structured_output(
model: str,
thinking_requested: bool,
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> bool:
return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) or (
thinking_requested and _bedrock_native_output_opt_in(model, custom_llm_provider, model_info)
)
BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64
def bedrock_cleaned_output_schema(
model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock"
) -> Mapping[str, object] | None:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if (
not _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
and schema_property_count(schema) > BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
):
return None
return AnthropicConfig.native_output_schema(schema)
def bedrock_native_output_schema(
model: str,
schema: Mapping[str, object],
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> Mapping[str, object] | None:
native_schema: Final = bedrock_cleaned_output_schema(model, schema, custom_llm_provider)
if _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider):
return schema if native_schema is None else native_schema
if _bedrock_native_output_opt_in(model, custom_llm_provider, model_info):
return native_schema
return None
def bedrock_native_output_format(
model: str,
output_format: Mapping[str, object],
custom_llm_provider: str = "bedrock",
model_info: Mapping[str, object] | None = None,
) -> Mapping[str, object] | None:
schema: Final = output_format.get("schema")
if not isinstance(schema, Mapping):
return (
output_format if bedrock_supports_native_structured_output(model, custom_llm_provider, model_info) else None
)
native_schema: Final = bedrock_native_output_schema(model, schema, custom_llm_provider, model_info)
if native_schema is None:
return None
return {**output_format, "schema": native_schema}
def apply_bedrock_invoke_structured_output(
model: str,
request_body: dict[str, object], # mutable-ok: edited in place like siblings
model_info: Mapping[str, object] | None = None,
) -> None:
"""
Route Anthropic structured-output params to what the Bedrock model supports.
@ -277,17 +362,18 @@ def apply_bedrock_invoke_structured_output(
if schema_format is None:
return
if _bedrock_model_supports(model, "supports_native_structured_output"):
native_format: Final = bedrock_native_output_format(model, schema_format, model_info=model_info)
if native_format is not None:
existing_output_config: Final = request_body.get("output_config")
if isinstance(existing_output_config, dict):
existing_output_config["format"] = schema_format
existing_output_config["format"] = native_format
else:
request_body["output_config"] = {"format": schema_format} # rebind-ok: out-param
request_body["output_config"] = {"format": native_format} # rebind-ok: out-param
return
verbose_logger.warning(
"Bedrock Invoke: model=%s does not advertise `supports_native_structured_output` "
"in model_prices_and_context_window.json, so the JSON schema was inlined into "
"Bedrock Invoke: model=%s does not take native structured output for this schema "
"per model_prices_and_context_window.json, so the JSON schema was inlined into "
"the last user message and is NOT enforced by the model.",
model,
)
@ -1296,7 +1382,6 @@ def is_claude_4_5_on_bedrock(model: str) -> bool:
_BEDROCK_MODEL_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
_DEPLOYMENT_MODEL_INFO: Final = TypeAdapter(dict[str, object])
def bedrock_converse_supports_strict_tools(model: str) -> bool:
@ -1327,8 +1412,7 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[
entry of ``model`` itself. A model nobody flagged keeps its schema as sent.
"""
params: Final = litellm_params or {}
model_info: Final = _DEPLOYMENT_MODEL_INFO.validate_python(params.get("model_info") or {})
deployment_flag: Final = model_info.get("supports_regex_lookaround")
deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround")
if isinstance(deployment_flag, bool):
return deployment_flag
base_model: Final = params.get("base_model")

9
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
View file

@ -16,7 +16,13 @@ from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import LiteLLMPydanticObjectBase, LlmProviders, TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import (
CountContextManagement,
LiteLLMPydanticObjectBase,
LlmProviders,
TokenCountMessagesFormat,
TokenCountResponse,
)
RUNTIME_TOKENIZER_TYPE: Final = "bedrock_api"
MANTLE_TOKENIZER_TYPE: Final = "bedrock_mantle_api"
@ -109,6 +115,7 @@ class BedrockTokenCounter(BaseTokenCounter):
system: object | None = None,
messages_format: TokenCountMessagesFormat = "anthropic",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
if not messages:
return None

19
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -3,6 +3,7 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -69,6 +70,9 @@ else:
LiteLLMLoggingObj = Any
_MODEL_INFO: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
class AmazonAnthropicClaudeMessagesConfig(
AnthropicMessagesConfig,
AmazonInvokeConfig,
@ -90,6 +94,9 @@ class AmazonAnthropicClaudeMessagesConfig(
def beta_headers_provider(self) -> str:
return self.custom_llm_provider or "bedrock"
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return True
BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(BedrockInvokeAnthropicMessagesRequest.__annotations__.keys())
def get_error_class(
@ -552,8 +559,17 @@ class AmazonAnthropicClaudeMessagesConfig(
if anthropic_model_info.is_eager_input_streaming_used(tools):
beta_set.add(ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER)
dangerous_tool_use_beta: Final = ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value
if anthropic_messages_optional_request_params.get("safeguards") is not None:
beta_set.add(ANTHROPIC_BETA_HEADER_VALUES.DANGEROUS_TOOL_USE_2026_09_03.value)
beta_set.add(dangerous_tool_use_beta)
elif dangerous_tool_use_beta in beta_set and self.drops_dangerous_tool_use_beta_without_safeguards():
beta_set.discard(dangerous_tool_use_beta)
verbose_logger.debug(
"Bedrock Invoke: dropping %s from client headers because the request has no safeguards. Bedrock "
"answers it with 'invalid beta flag' next to structured outputs before Opus 4.7, and on Sonnet 4.5, "
"Opus 4.5, and Opus 4.6 even alone.",
dangerous_tool_use_beta,
)
self._filter_context_management_for_bedrock_invoke(
anthropic_messages_request=anthropic_messages_request,
@ -739,6 +755,7 @@ class AmazonAnthropicClaudeMessagesConfig(
apply_bedrock_invoke_structured_output(
model=model,
request_body=anthropic_messages_request,
model_info=_MODEL_INFO.validate_python(getattr(litellm_params, "model_info", None)),
)
normalize_bedrock_opus_output_config_effort(
model=model,

3
litellm/llms/bedrock/messages/mantle_transformation.py
View file

@ -67,6 +67,9 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
def should_filter_anthropic_beta_headers(self) -> bool:
return False
def drops_dangerous_tool_use_beta_without_safeguards(self) -> bool:
return False
def get_complete_url(
self,
api_base: str | None,

878
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -1,18 +1,32 @@
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias
import base64
import copy
import json
import struct
from binascii import crc32
from collections.abc import AsyncGenerator, AsyncIterable, Callable, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, TypeAlias, TypeVar
from pydantic import TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
StreamTransformSink,
)
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
)
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.pass_through.guardrail_translation.handler import (
PassThroughEndpointHandler,
@ -21,6 +35,7 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
_INVOKE_STREAM_ACTION: Final = "invoke-with-response-stream"
_EVENT_STREAM_CONTENT_TYPE: Final = "vnd.amazon.eventstream"
_EVENT_STREAM_MEDIA_TYPE: Final = "application/vnd.amazon.eventstream"
@ -30,6 +45,10 @@ def _is_converse_endpoint(endpoint: str) -> bool:
return bool(parts) and parts[-1] in _CONVERSE_ACTIONS
def _is_invoke_stream_endpoint(endpoint: str) -> bool:
return endpoint.rstrip("/").split("/")[-1] == _INVOKE_STREAM_ACTION
def _generic_passthrough_handler() -> "PassThroughEndpointHandler":
"""
Fallback for non-Converse Bedrock routes (e.g. invoke). The generic
@ -217,8 +236,6 @@ class _StreamFrame(TypedDict):
def _unpack_uint32(buffer: bytes) -> int:
import struct
return struct.unpack("!I", buffer)[0]
@ -253,6 +270,578 @@ def _collect_stream_delta_text_holders(delta: object) -> list[_DeltaHolder]:
return holders
def _delta_frame_texts(payload: Mapping[str, object]) -> tuple[tuple[_StreamGroupKey, str], ...]:
"""The guardrailable texts of one ``contentBlockDelta`` payload, each keyed by its content block and kind."""
block_index: Final = payload.get("contentBlockIndex")
# Group per content block: a guardrail edit that changes length must not move text between blocks.
return tuple(
((block_index if isinstance(block_index, int) else None, group_key), container[key])
for group_key, container, key in _collect_stream_delta_text_holders(payload.get("delta"))
)
@dataclass(frozen=True, slots=True)
class _StreamTextGroups:
"""Stream texts joined per group, keeping where each fragment came from so a rewrite can be split back."""
groups: tuple[_StreamGroupKey, ...]
members: Mapping[_StreamGroupKey, Sequence[tuple[int, int]]]
fragments: Mapping[_StreamGroupKey, Sequence[str]]
def texts(self) -> tuple[str, ...]:
return tuple("".join(self.fragments[group]) for group in self.groups)
def _group_stream_texts(frame_texts: Sequence[Sequence[tuple[_StreamGroupKey, str]]]) -> _StreamTextGroups:
"""Group the per-frame texts by key in first-seen order; ``groups`` keeps only the groups with any text."""
order: Final[list[_StreamGroupKey]] = [] # mutable-ok: built in one pass over the frames
members: Final[dict[_StreamGroupKey, list[tuple[int, int]]]] = {} # mutable-ok: built in one pass over the frames
fragments: Final[dict[_StreamGroupKey, list[str]]] = {} # mutable-ok: built in one pass over the frames
for frame_idx, texts in enumerate(frame_texts):
for local_idx, (group_key, text) in enumerate(texts):
if group_key not in members:
members[group_key] = []
fragments[group_key] = []
order.append(group_key)
members[group_key].append((frame_idx, local_idx))
fragments[group_key].append(text)
return _StreamTextGroups(
groups=tuple(group for group in order if "".join(fragments[group])),
members=members,
fragments=fragments,
)
def _split_group_texts(groups: _StreamTextGroups, rewritten_texts: Sequence[str]) -> Mapping[tuple[int, int], str]:
"""Cut each group's rewritten text back into its fragments, sized like the original fragments."""
split: Final[dict[tuple[int, int], str]] = {} # mutable-ok: built in one pass over the groups
for group_key, rewritten in zip(groups.groups, rewritten_texts):
members = groups.members[group_key]
originals = groups.fragments[group_key]
total_orig = sum(len(t) for t in originals) or 1
pos = 0
for k, member in enumerate(members):
if k == len(members) - 1:
split[member] = rewritten[pos:]
else:
end = pos + round(len(rewritten) * len(originals[k]) / total_orig)
split[member] = rewritten[pos:end]
pos = end
return split
def _event_stream_message(headers_bytes: bytes, payload: bytes) -> bytes:
"""One AWS event-stream message: prelude, prelude CRC, headers, payload and message CRC."""
prelude: Final = struct.pack("!II", 12 + len(headers_bytes) + len(payload) + 4, len(headers_bytes))
prelude_crc: Final = crc32(prelude) & 0xFFFFFFFF
message: Final = prelude + struct.pack("!I", prelude_crc) + headers_bytes + payload
return message + struct.pack("!I", crc32(message) & 0xFFFFFFFF)
def _string_header(name: str, value: str) -> bytes:
name_bytes: Final = name.encode()
value_bytes: Final = value.encode()
return bytes((len(name_bytes),)) + name_bytes + b"\x07" + struct.pack("!H", len(value_bytes)) + value_bytes
def _string_headers(headers: Mapping[str, str]) -> bytes:
return b"".join(_string_header(name, value) for name, value in headers.items())
def _converse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
return _event_stream_message(
_string_headers({":event-type": event_type, ":content-type": "application/json", ":message-type": "event"}),
json.dumps(payload, separators=(",", ":")).encode(),
)
def _converse_exception(message: str, status_code: int) -> bytes:
"""An in-stream exception frame, which boto3 raises as ``EventStreamError`` carrying ``message``."""
return _event_stream_message(
_string_headers(
{
":exception-type": "validationException" if status_code < 500 else "internalServerException",
":content-type": "application/json",
":message-type": "exception",
}
),
json.dumps({"message": message}, separators=(",", ":")).encode(),
)
def _base64_json(event: Mapping[str, object]) -> str:
return base64.b64encode(json.dumps(event, separators=(",", ":")).encode()).decode()
def _invoke_chunk(event: Mapping[str, object]) -> bytes:
return _event_stream_message(
_string_headers({":event-type": "chunk", ":content-type": "application/json", ":message-type": "event"}),
json.dumps({"bytes": _base64_json(event)}, separators=(",", ":")).encode(),
)
@dataclass(slots=True)
class ConverseStreamFrame:
"""One frame of a Converse event stream as the proxy relays it.
``payload`` is the parsed JSON of an event frame; exception frames and bytes that do not decode keep it None and
are relayed as received. ``ends_stream`` marks the empty item the framer adds once the upstream has ended, so an
exception frame or a closed connection ends the stream as ``messageStop`` does. ``guardrailed_payload`` is set
when a guardrail rewrote the frame's text."""
raw: bytes
event_type: str | None = None
payload: Mapping[str, object] | None = None
guardrailed_payload: Mapping[str, object] | None = None
ends_stream: bool = False
@property
def block_index(self) -> int | None:
index: Final = self.payload.get("contentBlockIndex") if self.payload is not None else None
return index if isinstance(index, int) else None
def scanned_texts(self) -> tuple[tuple[_StreamGroupKey, str], ...]:
current: Final = self.guardrailed_payload if self.guardrailed_payload is not None else self.payload
return _delta_frame_texts(current) if self.event_type == "contentBlockDelta" and current is not None else ()
def received_texts(self) -> tuple[tuple[_StreamGroupKey, str], ...]:
return (
_delta_frame_texts(self.payload)
if self.event_type == "contentBlockDelta" and self.payload is not None
else ()
)
def encode(self) -> bytes:
if self.guardrailed_payload is None:
return self.raw
headers_length: Final = _unpack_uint32(self.raw[4:8])
return _event_stream_message(
self.raw[12 : 12 + headers_length], json.dumps(self.guardrailed_payload, separators=(",", ":")).encode()
)
def _anthropic_sse(event: Mapping[str, object] | None) -> bytes:
if event is None:
return b""
return f"event: {event['type']}\ndata: {json.dumps(event, separators=(',', ':'))}\n\n".encode()
@dataclass(slots=True)
class InvokeStreamFrame:
"""One frame of an invoke-with-response-stream as the proxy relays it.
``event`` is the Anthropic Messages stream event a ``chunk`` frame carries base64 encoded in the ``bytes`` of its
``payload``. Exception frames and bytes that do not decode keep it None and are relayed as received.
``ends_stream`` marks the empty item the framer adds once the upstream has ended, and ``guardrailed_event`` is set
when a guardrail rewrote the event."""
raw: bytes
payload: Mapping[str, object] | None = None
event: Mapping[str, object] | None = None
guardrailed_event: Mapping[str, object] | None = None
ends_stream: bool = False
def sse(self) -> bytes:
"""The event as a ``/v1/messages`` stream carries it, the shape its guardrail translation scans."""
return _anthropic_sse(self.guardrailed_event if self.guardrailed_event is not None else self.event)
def received_sse(self) -> bytes:
"""The event as received from the upstream, before any guardrail rewrote it."""
return _anthropic_sse(self.event)
def encode(self) -> bytes:
if self.guardrailed_event is None or self.payload is None:
return self.raw
headers_length: Final = _unpack_uint32(self.raw[4:8])
return _event_stream_message(
self.raw[12 : 12 + headers_length],
json.dumps({**self.payload, "bytes": _base64_json(self.guardrailed_event)}, separators=(",", ":")).encode(),
)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_Frame: Final = TypeVar("_Frame", ConverseStreamFrame, InvokeStreamFrame)
def _event_message(raw: bytes) -> tuple[str, bytes] | None:
"""The event type and payload of one event-stream frame, or None for an exception frame or bytes that do not
decode."""
from botocore.eventstream import EventStreamBuffer, ParserError
buffer: Final = EventStreamBuffer()
buffer.add_data(raw)
try:
message: Final = next(iter(buffer))
except (ParserError, StopIteration) as e:
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: could not decode event-stream frame, relaying it unmodified: %s", e
)
return None
# botocore-stubs types EventStreamMessage.headers as int; at runtime it is the decoded header dict.
headers: Final = _JSON_OBJECT.validate_python(message.headers)
event_type: Final = headers.get(":event-type")
if headers.get(":message-type") != "event" or not isinstance(event_type, str):
return None
return event_type, message.payload
def _decode_converse_frame(raw: bytes) -> ConverseStreamFrame:
message: Final = _event_message(raw)
if message is None:
return ConverseStreamFrame(raw=raw)
event_type, payload_bytes = message
try:
payload: Final = _JSON_OBJECT.validate_python(json.loads(payload_bytes))
except (json.JSONDecodeError, UnicodeDecodeError, ValidationError):
return ConverseStreamFrame(raw=raw, event_type=event_type)
return ConverseStreamFrame(raw=raw, event_type=event_type, payload=payload)
def _decode_invoke_frame(raw: bytes) -> InvokeStreamFrame:
message: Final = _event_message(raw)
if message is None or message[0] != "chunk":
return InvokeStreamFrame(raw=raw)
try:
payload: Final = _JSON_OBJECT.validate_python(json.loads(message[1]))
encoded: Final = payload.get("bytes")
event: Final = (
_JSON_OBJECT.validate_python(json.loads(base64.b64decode(encoded))) if isinstance(encoded, str) else None
)
except (ValueError, ValidationError):
return InvokeStreamFrame(raw=raw)
if event is None or not isinstance(event.get("type"), str):
return InvokeStreamFrame(raw=raw)
return InvokeStreamFrame(raw=raw, payload=payload, event=event)
async def _event_stream_frames(
chunks: AsyncIterable[bytes], frame_type: type[_Frame], decode: Callable[[bytes], _Frame]
) -> AsyncGenerator[_Frame, None]:
"""Cut the upstream bytes into frames as soon as each frame has fully arrived, then add an empty item that ends
the stream once the upstream has ended.
Bytes that cannot be framed (a length below the 16-byte minimum, or a stream that ends inside a frame) are relayed
as received."""
pending = b"" # rebind-ok: bytes received but not yet framed
framing = True # rebind-ok: cleared once a frame length is unusable
async for chunk in chunks:
if not framing:
yield frame_type(raw=chunk)
continue
pending += chunk
while len(pending) >= 4:
total_length = _unpack_uint32(pending[:4])
if total_length < 16:
framing = False
break
if len(pending) < total_length:
break
yield decode(pending[:total_length])
pending = pending[total_length:]
if not framing and pending:
yield frame_type(raw=pending)
pending = b""
if pending:
yield frame_type(raw=pending)
yield frame_type(raw=b"", ends_stream=True)
def _converse_frames(items: Sequence[object]) -> tuple[ConverseStreamFrame, ...]:
"""The frames among ``items``, with the bytes an earlier guardrail's block message added read back into frames."""
return tuple(
item if isinstance(item, ConverseStreamFrame) else _decode_converse_frame(item)
for item in items
if isinstance(item, (ConverseStreamFrame, bytes))
)
def _reads_stream_marks(guardrail: "CustomGuardrail") -> bool:
return getattr(guardrail, "reads_stream_marks", False) is True
def _json_object_member(value: object, key: str) -> Mapping[str, object] | None:
"""``value[key]`` when both are JSON objects, else None."""
try:
return _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_python(value).get(key))
except ValidationError:
return None
def _reject_json_constant(name: str) -> object:
raise ValueError(f"{name} is not JSON")
def _finished_tool_input(text: str) -> object:
"""A tool call's streamed input as the JSON document Converse returns, or the text as received when it is not
JSON (cut short, rewritten by a guardrail, or holding NaN or nesting too deep to send on)."""
try:
return json.loads(text, parse_constant=_reject_json_constant) if text else {}
except (ValueError, RecursionError):
return text
def _finished_converse_content(
frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups
) -> list[dict[str, object]]: # mutable-ok: the content of the finished response guardrails receive
"""The finished reply's text, reasoning and tool-call blocks, shaped as a non-streaming Converse response
holds them."""
texts: Final = dict(zip(groups.groups, groups.texts()))
tool_starts: Final[dict[int | None, Mapping[str, object]]] = {} # mutable-ok: built in one pass over the frames
keys: Final[dict[_StreamGroupKey, None]] = {} # mutable-ok: blocks in stream order, built in one pass
for frame in frames:
tool_use = (
_json_object_member(_json_object_member(frame.payload, "start"), "toolUse")
if frame.event_type == "contentBlockStart"
else None
)
if tool_use is not None:
tool_starts[frame.block_index] = tool_use
keys[(frame.block_index, "tool")] = None
keys.update((key, None) for key, _ in frame.scanned_texts())
content: Final[list[dict[str, object]]] = [] # mutable-ok: built in one pass over the blocks
for block_index, kind in keys:
text = texts.get((block_index, kind), "")
if kind == "tool":
tool_start = tool_starts.get(block_index, {})
named = {name: tool_start[name] for name in ("toolUseId", "name", "type") if name in tool_start}
content.append({"toolUse": {**named, "input": _finished_tool_input(text)}})
elif text and kind == "reasoning":
content.append({"reasoningContent": {"reasoningText": {"text": text}}})
elif text:
content.append({"text": text})
return content
def _stream_marked_request_data(
request_data: Mapping[str, object], frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups, ended: bool
) -> dict[str, object]: # mutable-ok: the request dict guardrails receive
"""``request_data`` with the marks chat completions gives a streamed answer: ``stream`` on every scan and, once
the stream has ended, the finished ``response``."""
if not ended:
return {**request_data, "stream": True}
stop_reason: Final = next(
(
frame.payload.get("stopReason")
for frame in frames
if frame.event_type == "messageStop" and frame.payload is not None
),
None,
)
message: Final = {"role": "assistant", "content": _finished_converse_content(frames, groups)}
response: Final = {"output": {"message": message}, **({"stopReason": stop_reason} if stop_reason else {})}
return {**request_data, "stream": True, "response": response}
def _invoke_frames(items: Sequence[object]) -> tuple[InvokeStreamFrame, ...]:
return tuple(item for item in items if isinstance(item, InvokeStreamFrame))
def _sse_events(sse: bytes) -> tuple[Mapping[str, object], ...]:
return tuple(
_JSON_OBJECT.validate_python(json.loads(line[len("data:") :]))
for line in sse.decode().splitlines()
if line.startswith("data:")
)
def _invoke_message_stopped(frames: Sequence[InvokeStreamFrame]) -> bool:
"""Whether a ``message_delta`` with a ``stop_reason`` came, as a ``/v1/messages`` stream ends."""
return any(
frame.event is not None
and frame.event.get("type") == "message_delta"
and isinstance(delta := frame.event.get("delta"), Mapping)
and _JSON_OBJECT.validate_python(delta).get("stop_reason") is not None
for frame in frames
)
def _invoke_stream_has_ended(frames: Sequence[InvokeStreamFrame]) -> bool:
"""Ended on a ``message_delta`` with a ``stop_reason`` or once the upstream has ended."""
return any(frame.ends_stream for frame in frames) or _invoke_message_stopped(frames)
_UPSTREAM_ENDED_SSE: Final = _anthropic_sse(
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": 0},
}
)
def _invoke_scan_sse(frames: Sequence[InvokeStreamFrame], events: Sequence[bytes]) -> tuple[bytes, ...]:
"""The ``/v1/messages`` events to scan for these frames, closed with a ``message_delta`` when the upstream ended
without one so the translation scans them as a whole stream, tool calls included."""
if any(frame.ends_stream for frame in frames) and not _invoke_message_stopped(frames):
return (*events, _UPSTREAM_ENDED_SSE)
return tuple(events)
async def _scan_invoke_stream(
frames: Sequence[InvokeStreamFrame],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"],
user_api_key_dict: Optional["UserAPIKeyAuth"],
request_data: dict[str, object] | None, # mutable-ok: the request dict guardrails receive
) -> None:
"""Scan the Anthropic events relayed so far as ``/v1/messages`` scans its stream, with the request marked as
streaming and those events as its ``responses``. Once the stream has ended, the rewrites are written back into the
frames, which reach the caller only when the stream is held until moderated."""
received: Final = tuple(frame.sse() for frame in frames)
scan_events: Final = _invoke_scan_sse(frames, received)
scanned: Final[list[object]] = list(scan_events) # mutable-ok: the /v1/messages translation rewrites it in place
if request_data is not None:
request_data["stream"] = True # rebind-ok: marks the request guardrails receive as a stream
request_data["responses"] = scanned # rebind-ok: each scan sees every event relayed so far
await AnthropicMessagesHandler().process_output_streaming_response( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
responses_so_far=scanned,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
deliver_ended_stream_rewrites=_invoke_stream_has_ended(frames),
)
for frame, before, after in zip(frames, received, scanned):
if isinstance(after, bytes) and after != before and (rewritten := _sse_events(after)):
frame.guardrailed_event = rewritten[0]
def _invoke_block_frames(
exc: "ModifyResponseException", stream_started: bool, frames: Sequence[InvokeStreamFrame]
) -> tuple[bytes, ...]:
"""The ``/v1/messages`` block events for the stream relayed so far, each in its own ``chunk`` frame."""
sse_chunks: Final = AnthropicMessagesHandler().build_block_sse_chunks(
exc, stream_started=stream_started, responses_so_far=[frame.sse() for frame in frames]
)
return tuple(_invoke_chunk(event) for event in _sse_events(b"".join(sse_chunks)))
def _write_back_stream_texts(
frames: Sequence[ConverseStreamFrame], groups: _StreamTextGroups, rewritten_texts: Sequence[str]
) -> None:
split: Final = _split_group_texts(groups, rewritten_texts)
for frame_idx, frame in enumerate(frames):
current = frame.guardrailed_payload if frame.guardrailed_payload is not None else frame.payload
if frame.event_type != "contentBlockDelta" or current is None:
continue
rewritten = copy.deepcopy(current)
changed = False
for local_idx, (_, container, key) in enumerate(_collect_stream_delta_text_holders(rewritten.get("delta"))):
new_text = split.get((frame_idx, local_idx))
if new_text is not None and new_text != container[key]:
container[key] = new_text
changed = True
if changed:
frame.guardrailed_payload = rewritten
_INVOKE_DELTA_TEXT_FIELDS: Final = MappingProxyType(
{
"text_delta": ("text", "text"),
"input_json_delta": ("tool", "partial_json"),
"thinking_delta": ("reasoning", "thinking"),
}
)
@dataclass(frozen=True, slots=True)
class _InvokeDeltaText:
"""The guardrailable text one Anthropic event's delta carries, keyed by its content block and kind like the
Converse holders: text, tool input and thinking. Signatures and citations are never rewritten."""
group: _StreamGroupKey
delta: Mapping[str, object]
field: str
text: str
def _invoke_delta_text(event: Mapping[str, object] | None) -> _InvokeDeltaText | None:
if event is None:
return None
try:
delta: Final = _JSON_OBJECT.validate_python(event.get("delta"))
except ValidationError:
return None
kind: Final = delta.get("type")
field: Final = _INVOKE_DELTA_TEXT_FIELDS.get(kind) if isinstance(kind, str) else None
text: Final = delta.get(field[1]) if field is not None else None
if field is None or not isinstance(text, str):
return None
index: Final = event.get("index")
return _InvokeDeltaText((index if isinstance(index, int) else None, field[0]), delta, field[1], text)
async def _post_call_rewritten_texts(
groups: _StreamTextGroups,
proxy_logging_obj: "ProxyLogging",
user_api_key_dict: "UserAPIKeyAuth",
data: dict[str, object], # mutable-ok: the request dict post_call hooks receive
) -> Sequence[str] | None:
"""Run the post_call guardrails on a buffered stream's texts as one Converse response with a content block per
group, and return each group's text as the guardrails left it, or None when the response came back in another
shape."""
synthetic_response: Final[dict] = {
"output": {
"message": {
"role": "assistant",
"content": [{"text": text} for text in groups.texts()],
}
},
"stopReason": "end_turn",
}
processed: Final = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=synthetic_response,
)
if not isinstance(processed, dict):
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: post_call_success_hook returned %s, leaving event stream unmodified",
type(processed).__name__,
)
return None
try:
processed_blocks: Final = processed["output"]["message"]["content"]
return [processed_blocks[i]["text"] for i in range(len(groups.groups))]
except (KeyError, IndexError, TypeError):
return None
async def _guardrail_buffered_invoke_stream(
body_bytes: bytes,
proxy_logging_obj: "ProxyLogging",
user_api_key_dict: "UserAPIKeyAuth",
data: dict[str, object], # mutable-ok: the request dict post_call hooks receive
) -> bytes:
"""Scan a whole Claude invoke stream the way the buffered Converse path scans its stream, and write the guardrail
rewrites back into the ``chunk`` frames that carried each text."""
async def received() -> AsyncGenerator[bytes, None]:
yield body_bytes
frames: Final = [frame async for frame in _event_stream_frames(received(), InvokeStreamFrame, _decode_invoke_frame)]
texts: Final = [_invoke_delta_text(frame.event) for frame in frames]
groups: Final = _group_stream_texts([[] if text is None else [(text.group, text.text)] for text in texts])
if not groups.groups:
return body_bytes
rewritten_texts: Final = await _post_call_rewritten_texts(groups, proxy_logging_obj, user_api_key_dict, data)
if rewritten_texts is None:
return body_bytes
split: Final = _split_group_texts(groups, rewritten_texts)
for frame_idx, (frame, text) in enumerate(zip(frames, texts)):
if (
text is not None
and frame.event is not None
and (new_text := split.get((frame_idx, 0))) is not None
and new_text != text.text
):
frame.guardrailed_event = {**frame.event, "delta": {**text.delta, text.field: new_text}}
return b"".join(frame.encode() for frame in frames)
class BedrockPassthroughGuardrailHandler(BaseTranslation):
@staticmethod
def is_event_stream_content_type(content_type: str) -> bool:
@ -263,19 +852,32 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
return _EVENT_STREAM_MEDIA_TYPE
@staticmethod
def event_stream_endpoint_is_de_anonymizable(endpoint: str) -> bool:
return _is_converse_endpoint(endpoint)
def event_stream_endpoint_is_de_anonymizable(endpoint: str, model: str, body: object) -> bool:
"""The streams the buffered path can scan whole and rewrite, the same ones the relay scans."""
return BedrockPassthroughGuardrailHandler.event_stream_endpoint_is_relayable(endpoint, model, body)
@staticmethod
def event_stream_endpoint_is_relayable(endpoint: str, model: str, body: object) -> bool:
"""Converse streams of every model, and invoke streams of Claude, whose chunks carry Anthropic Messages stream
events. Claude is told by its model id or, for ids that do not name it such as application inference profile
ARNs, by the ``anthropic_version`` Bedrock requires in every Claude Messages request body."""
return _is_converse_endpoint(endpoint) or (
_is_invoke_stream_endpoint(endpoint)
and (
BaseAWSLLM.get_bedrock_invoke_provider(model) == "anthropic"
or (isinstance(body, Mapping) and "anthropic_version" in body)
)
)
@staticmethod
async def de_anonymize_event_stream(
body_bytes: bytes,
proxy_logging_obj: "ProxyLogging",
user_api_key_dict: "UserAPIKeyAuth",
data: dict,
data: dict[str, object], # mutable-ok: the request dict post_call hooks receive
) -> bytes:
import json as _json
import struct
from binascii import crc32 as esm_crc32
if _is_invoke_stream_endpoint(str(data.get("endpoint") or "")):
return await _guardrail_buffered_invoke_stream(body_bytes, proxy_logging_obj, user_api_key_dict, data)
from botocore.eventstream import EventStreamBuffer
@ -304,16 +906,11 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
frames.append({"raw": frame_raw, "texts": []})
continue
texts: list[tuple[_StreamGroupKey, str]] = []
texts: Sequence[tuple[_StreamGroupKey, str]] = ()
if event_type == "contentBlockDelta":
try:
payload_dict: dict[str, object] = _json.loads(payload_bytes)
block_index = payload_dict.get("contentBlockIndex")
# Group per content block: a guardrail edit that changes length must not move text between blocks.
texts = [
((block_index if isinstance(block_index, int) else None, group_key), container[key])
for group_key, container, key in _collect_stream_delta_text_holders(payload_dict.get("delta"))
]
payload_dict: dict[str, object] = json.loads(payload_bytes)
texts = _delta_frame_texts(payload_dict)
except Exception as e:
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: could not parse "
@ -325,66 +922,17 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
trailing_bytes: Final = body_bytes[offset:]
group_order: Final[list[_StreamGroupKey]] = []
group_members: Final[dict[_StreamGroupKey, list[tuple[int, int]]]] = {}
group_texts: Final[dict[_StreamGroupKey, list[str]]] = {}
for frame_idx, frame in enumerate(frames):
for local_idx, (group_key, text) in enumerate(frame["texts"]):
if group_key not in group_members:
group_members[group_key] = []
group_texts[group_key] = []
group_order.append(group_key)
group_members[group_key].append((frame_idx, local_idx))
group_texts[group_key].append(text)
active_groups: Final = [gk for gk in group_order if "".join(group_texts[gk])]
if not active_groups:
groups: Final = _group_stream_texts([frame["texts"] for frame in frames])
if not groups.groups:
return body_bytes
synthetic_response: Final[dict] = {
"output": {
"message": {
"role": "assistant",
"content": [{"text": "".join(group_texts[gk])} for gk in active_groups],
}
},
"stopReason": "end_turn",
}
processed: Final = await proxy_logging_obj.post_call_success_hook(
data=data,
user_api_key_dict=user_api_key_dict,
response=synthetic_response,
de_anonymized_texts: Final = await _post_call_rewritten_texts(
groups, proxy_logging_obj, user_api_key_dict, data
)
if not isinstance(processed, dict):
verbose_proxy_logger.debug(
"BedrockPassthroughGuardrailHandler: post_call_success_hook returned %s, "
"leaving event stream unmodified",
type(processed).__name__,
)
if de_anonymized_texts is None:
return body_bytes
try:
processed_blocks: Final = processed["output"]["message"]["content"]
de_anonymized_texts: Final = [processed_blocks[i]["text"] for i in range(len(active_groups))]
except (KeyError, IndexError, TypeError):
return body_bytes
new_text_map: Final[dict[tuple[int, int], str]] = {}
for group_key, de_anonymized_text in zip(active_groups, de_anonymized_texts):
members = group_members[group_key]
orig_texts = group_texts[group_key]
total_orig = sum(len(t) for t in orig_texts) or 1
de_anon_len = len(de_anonymized_text)
pos = 0
for k, member in enumerate(members):
if k == len(members) - 1:
new_text_map[member] = de_anonymized_text[pos:]
else:
end = pos + round(de_anon_len * len(orig_texts[k]) / total_orig)
new_text_map[member] = de_anonymized_text[pos:end]
pos = end
new_text_map: Final = _split_group_texts(groups, de_anonymized_texts)
result_parts: Final[list[bytes]] = []
@ -399,27 +947,19 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
headers_bytes = frame_raw[12 : 12 + orig_hdrs_len]
try:
payload_dict = _json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
payload_dict = json.loads(frame_raw[12 + orig_hdrs_len : orig_total - 4])
for local_idx, (_, container, key) in enumerate(
_collect_stream_delta_text_holders(payload_dict.get("delta"))
):
new_text = new_text_map.get((frame_idx, local_idx))
if new_text is not None:
container[key] = new_text
new_payload = _json.dumps(payload_dict, separators=(",", ":")).encode()
new_payload = json.dumps(payload_dict, separators=(",", ":")).encode()
except Exception:
result_parts.append(frame_raw)
continue
new_total = 12 + orig_hdrs_len + len(new_payload) + 4
prelude = struct.pack("!II", new_total, orig_hdrs_len)
prelude_crc_val = esm_crc32(prelude) & 0xFFFFFFFF
prelude_crc_b = struct.pack("!I", prelude_crc_val)
part_for_msg_crc = prelude_crc_b + headers_bytes + new_payload
msg_crc_val = esm_crc32(part_for_msg_crc, prelude_crc_val) & 0xFFFFFFFF
msg_crc_b = struct.pack("!I", msg_crc_val)
result_parts.append(prelude + prelude_crc_b + headers_bytes + new_payload + msg_crc_b)
result_parts.append(_event_stream_message(headers_bytes, new_payload))
result_parts.append(trailing_bytes)
return b"".join(result_parts)
@ -477,8 +1017,13 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
request_data: dict | None = None,
) -> object:
endpoint: Final = (request_data or {}).get("endpoint", "")
if endpoint and not _is_converse_endpoint(endpoint):
endpoint: Final[object] = (request_data or {}).get("endpoint", "")
if (
isinstance(endpoint, str)
and endpoint
and not _is_converse_endpoint(endpoint)
and not _is_invoke_stream_endpoint(endpoint)
):
return await _generic_passthrough_handler().process_output_response(
response=response,
guardrail_to_apply=guardrail_to_apply,
@ -503,6 +1048,20 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
if not texts:
return response
await self._scan_output_texts(
texts, holders, guardrail_to_apply, litellm_logging_obj, user_api_key_dict, request_data
)
return response
async def _scan_output_texts(
self,
texts: list[str],
holders: list[_StringHolder],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"],
user_api_key_dict: Optional["UserAPIKeyAuth"],
request_data: dict[str, object] | None, # mutable-ok: the request dict guardrails receive
) -> None:
effective_request_data = request_data or {}
if "litellm_metadata" not in effective_request_data and user_api_key_dict is not None:
user_metadata: Final = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
@ -513,8 +1072,8 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
}
inputs: Final = GenericGuardrailAPIInputs(texts=texts)
model: Final = effective_request_data.get("model") if effective_request_data else None
if model:
model: Final = effective_request_data.get("model")
if isinstance(model, str) and model:
inputs["model"] = model
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
@ -528,4 +1087,141 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
if guardrailed_texts:
_write_back_texts(guardrailed_texts, holders)
return response
async def process_output_streaming_response(
self,
responses_so_far: Sequence[object],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
request_data: dict[str, object] | None = None, # mutable-ok: the request dict guardrails receive
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> Sequence[object]:
"""
Scan the Converse frames relayed so far as the buffered path scans a whole response: one text per content
block and kind. Once the stream has ended, guardrail rewrites are written back into the frames, which reach
the caller only when the stream is held until moderated. A guardrail that sets ``reads_stream_marks`` gets
a copy of the request marked as chat completions marks a streamed answer; the others get it unchanged.
Such a guardrail also gets the ended scan of a reply with no text, such as a call to a tool that takes no
input, with no texts to scan. Invoke frames are scanned as Anthropic Messages events.
"""
invoke_frames: Final = _invoke_frames(responses_so_far)
if invoke_frames:
await _scan_invoke_stream(
invoke_frames, guardrail_to_apply, litellm_logging_obj, user_api_key_dict, request_data
)
return responses_so_far
frames: Final = _converse_frames(responses_so_far)
groups: Final = _group_stream_texts([frame.scanned_texts() for frame in frames])
ended: Final = self._check_streaming_has_ended(frames)
reads_marks: Final = _reads_stream_marks(guardrail_to_apply)
if not groups.groups:
if request_data is not None and reads_marks and ended and _finished_converse_content(frames, groups):
await self._scan_output_texts(
[],
[],
guardrail_to_apply,
litellm_logging_obj,
user_api_key_dict,
_stream_marked_request_data(request_data, frames, groups, ended),
)
return responses_so_far
content: Final = [{"text": text} for text in groups.texts()]
await self.process_output_response( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
response={"output": {"message": {"role": "assistant", "content": content}}},
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=(
_stream_marked_request_data(request_data, frames, groups, ended)
if request_data is not None and reads_marks
else request_data
),
)
if ended:
_write_back_stream_texts(frames, groups, [block["text"] for block in content])
return responses_so_far
def needs_ended_stream_scan(self, guardrail: "CustomGuardrail", responses_so_far: Sequence[object]) -> bool:
return _reads_stream_marks(guardrail)
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
invoke_frames: Final = _invoke_frames(responses_so_far)
if invoke_frames:
return AnthropicMessagesHandler().get_streaming_scan_key(
list(_invoke_scan_sse(invoke_frames, [frame.received_sse() for frame in invoke_frames]))
)
frames: Final = _converse_frames(responses_so_far)
return StreamingScanKey(
texts=_group_stream_texts([frame.received_texts() for frame in frames]).texts(),
stream_ended=self._check_streaming_has_ended(frames),
)
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
return _invoke_stream_has_ended(_invoke_frames(responses_so_far)) or any(
frame.event_type == "messageStop" or frame.ends_stream for frame in _converse_frames(responses_so_far)
)
def build_block_sse_chunks(
self,
exc: "ModifyResponseException",
stream_started: bool = False,
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes]:
"""
End the message with the guardrail's message as a new text block and ``stopReason`` ``guardrail_intervened``,
closing any content block left open and starting the message when nothing was relayed yet. An invoke stream
gets the ``/v1/messages`` block events instead.
"""
invoke_frames: Final = _invoke_frames(responses_so_far or ())
if invoke_frames or _is_invoke_stream_endpoint(str(exc.request_data.get("endpoint") or "")):
return _invoke_block_frames(exc, stream_started, invoke_frames)
frames: Final = _converse_frames(responses_so_far or ())
started: Final = frozenset(
index
for index in (
frame.block_index for frame in frames if frame.event_type in ("contentBlockStart", "contentBlockDelta")
)
if index is not None
)
stopped: Final = frozenset(
index
for index in (frame.block_index for frame in frames if frame.event_type == "contentBlockStop")
if index is not None
)
block_index: Final = max(started | stopped, default=-1) + 1
return (
*(
()
if any(frame.event_type == "messageStart" for frame in frames)
else (_converse_event("messageStart", {"role": "assistant"}),)
),
*(_converse_event("contentBlockStop", {"contentBlockIndex": index}) for index in sorted(started - stopped)),
_converse_event("contentBlockDelta", {"contentBlockIndex": block_index, "delta": {"text": exc.message}}),
_converse_event("contentBlockStop", {"contentBlockIndex": block_index}),
_converse_event("messageStop", {"stopReason": "guardrail_intervened"}),
)
@staticmethod
def owns_event_stream_item(item: object) -> bool:
return isinstance(item, (ConverseStreamFrame, InvokeStreamFrame))
@staticmethod
def event_stream_items(
chunks: AsyncIterable[bytes], endpoint: str
) -> AsyncGenerator[ConverseStreamFrame | InvokeStreamFrame, None]:
if _is_invoke_stream_endpoint(endpoint):
return _event_stream_frames(chunks, InvokeStreamFrame, _decode_invoke_frame)
return _event_stream_frames(chunks, ConverseStreamFrame, _decode_converse_frame)
@staticmethod
def encode_event_stream_item(item: object) -> bytes:
if isinstance(item, (ConverseStreamFrame, InvokeStreamFrame)):
return item.encode()
if isinstance(item, bytes):
return item
raise TypeError(f"cannot relay {type(item).__name__} on a Bedrock event stream")
@staticmethod
def event_stream_error_item(message: str, status_code: int) -> bytes:
return _converse_exception(message, status_code)

65
litellm/llms/bedrock/passthrough/transformation.py
View file

@ -1,14 +1,18 @@
import json
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from functools import partial
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Optional, cast
import httpx
from httpx import Response
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
from litellm.types.utils import ModelResponseStream
from ..base_aws_llm import BaseAWSLLM
@ -91,6 +95,57 @@ def _translate_message(decoder: "AWSEventStreamDecoder", message: str) -> ModelR
return None
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_JSON_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def _leaf_texts(block: Mapping[str, object]) -> Iterator[str]:
"""The prompt text a content block carries itself: its text or JSON, or a tool call's input."""
text: Final = block.get("text")
if isinstance(text, str):
yield text
if "json" in block:
yield json.dumps(block["json"], default=str)
if "toolUse" in block:
yield json.dumps(_JSON_OBJECT.validate_python(block["toolUse"]).get("input"), default=str)
def _block_texts(block: Mapping[str, object]) -> Iterator[str]:
"""The prompt text a request content block carries, including the blocks of a tool result's content, which
Bedrock does not nest further."""
yield from _leaf_texts(block)
if "toolResult" not in block:
return
tool_result: Final = _JSON_OBJECT.validate_python(block["toolResult"])
yield from chain.from_iterable(map(_leaf_texts, _JSON_OBJECTS.validate_python(tool_result.get("content", ()))))
def _content_text(content: object) -> str:
if isinstance(content, str):
return content
return "\n".join(chain.from_iterable(map(_block_texts, _JSON_OBJECTS.validate_python(content))))
def _chat_message(turn: Mapping[str, object]) -> ChatCompletionUserMessage:
return ChatCompletionUserMessage(role="user", content=_content_text(turn.get("content", "")))
def _count_prompt_tokens(model: str, request_body: object) -> int:
"""Estimate a Bedrock request's prompt tokens from its system prompt and turns, the way a chat completions
stream that ended before the provider reported usage is estimated."""
from litellm.litellm_core_utils.token_counter import token_counter
body: Final = _JSON_OBJECT.validate_python(request_body)
if "messages" not in body:
return 0
system: Final = body.get("system")
messages: Final = (
*((ChatCompletionSystemMessage(role="system", content=_content_text(system)),) if system else ()),
*map(_chat_message, _JSON_OBJECTS.validate_python(body["messages"])),
)
return token_counter(model=model, messages=messages)
def _build_logged_response(
chunks: Sequence[ModelResponseStream], litellm_logging_obj: "LiteLLMLoggingObj"
) -> Optional["CostResponseTypes"]:
@ -98,7 +153,13 @@ def _build_logged_response(
if len(chunks) == 0:
return None
return stream_chunk_builder(chunks=list(chunks), logging_obj=litellm_logging_obj)
return stream_chunk_builder(
chunks=list(chunks),
logging_obj=litellm_logging_obj,
count_prompt_tokens=partial(
_count_prompt_tokens, litellm_logging_obj.model, litellm_logging_obj.model_call_details.get("input")
),
)
class BedrockEventStreamCollector:

93
litellm/llms/bedrock/realtime/handler.py
View file

@ -8,7 +8,7 @@ import asyncio
import contextlib
import importlib.metadata
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, NoReturn, Protocol, runtime_checkable
@ -21,7 +21,7 @@ from litellm._uuid import uuid
from litellm.constants import (
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY,
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
BEDROCK_REALTIME_SDK_DISTRIBUTION,
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
@ -37,9 +37,8 @@ from litellm.types.realtime import RealtimeResponseTransformInput
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
_CLIENT_MODALITIES_ADAPTER: Final[TypeAdapter["list[str] | None"]] = TypeAdapter(list[str] | None)
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
@ -177,25 +176,35 @@ async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message:
return
def _pending_session_update(scope: Mapping[str, object]) -> str | None:
"""A fallback attempt on the same websocket replays the session.update the failed attempt never acked."""
if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
raise BedrockError(
status_code=400,
message=(
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
),
)
pending: Final = scope.get(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY)
def _is_session_committed(scope: Mapping[str, object]) -> bool:
return scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True
def raise_if_session_committed(scope: Mapping[str, object] | None) -> None:
"""A fallback attempt on a websocket whose Bedrock stream already took input would silently start over."""
if scope is None or not _is_session_committed(scope):
return
committed_failure: Final = scope.get(BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY)
raise BedrockError(
status_code=400,
message=(
"Bedrock realtime session already committed to a provider stream; it cannot be replayed"
+ (f". The committed stream failed with: {committed_failure}" if committed_failure else "")
),
)
def _pending_opening_message(scope: Mapping[str, object]) -> str | None:
"""A fallback attempt on the same websocket replays the opening message Bedrock never accepted a stream for."""
raise_if_session_committed(scope)
pending: Final = scope.get(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY)
return pending if isinstance(pending, str) else None
def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn:
error: Final = _as_bedrock_error(failure)
verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error)))
if scope.get(BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY) is True:
if _is_session_committed(scope):
scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error))
raise error from failure
@ -207,26 +216,36 @@ def _parse_client_message(message: str) -> Mapping[str, JsonValue]:
return _EMPTY_JSON_OBJECT
async def _ack_session_update(
async def _send_opening_message(
client_ws: RealtimeClientWebSocket,
bedrock_stream: BedrockBidirectionalStream,
message: str,
bedrock_messages: Sequence[str],
send_to_bedrock: Callable[[str], Awaitable[None]],
) -> None:
"""Keep the opening message for a fallback to replay until Bedrock accepts the stream it opened."""
client_ws.scope[BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY] = message # rebind-ok: scope outlives attempts
for bedrock_message in bedrock_messages:
await send_to_bedrock(bedrock_message)
await bedrock_stream.await_output()
client_ws.scope.pop(BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, None)
client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True # rebind-ok: scope outlives the attempt
async def _ack_session_update(
client_ws: RealtimeClientWebSocket,
transformation_config: BedrockRealtimeConfig,
model: str,
logging_obj: LiteLLMLogging | None,
parsed_client_message: Mapping[str, JsonValue],
) -> bool:
"""Ack the client's session.update once Bedrock accepted the stream; False means the client is gone."""
await bedrock_stream.await_output()
client_ws.scope.pop(BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY, None)
client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True # rebind-ok: scope outlives the attempt
"""Ack the client's session.update; False means the client is gone."""
if logging_obj is None:
return True
requested_modalities: Final = _CLIENT_MODALITIES_ADAPTER.validate_python(
_json_dict(parsed_client_message.get("session")).get("modalities")
)
modalities: Final = requested_modalities(_json_dict(parsed_client_message.get("session")))
try:
await client_ws.send_text(
json.dumps(transformation_config.session_updated_event(model, logging_obj, requested_modalities))
json.dumps(transformation_config.session_updated_event(model, logging_obj, modalities))
)
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
@ -314,7 +333,7 @@ class BedrockRealtime(BaseAWSLLM):
except ImportError as e:
raise _sdk_import_error(self._sdk_version_lookup(), e) from e
pending_session_update: Final = _pending_session_update(websocket.scope)
pending_opening_message: Final = _pending_opening_message(websocket.scope)
# Get AWS region
if aws_region_name is None:
@ -376,7 +395,7 @@ class BedrockRealtime(BaseAWSLLM):
try:
await self._run_session(
websocket, open_bidirectional_stream, model, logging_obj, pending_session_update, drop_params
websocket, open_bidirectional_stream, model, logging_obj, pending_opening_message, drop_params
)
finally:
await _close_bedrock_client(bedrock_client)
@ -387,7 +406,7 @@ class BedrockRealtime(BaseAWSLLM):
open_bidirectional_stream: Callable[[], Awaitable[BedrockBidirectionalStream]],
model: str,
logging_obj: LiteLLMLogging,
pending_session_update: str | None,
pending_opening_message: str | None,
drop_params: bool = False,
) -> None:
transformation_config: Final = BedrockRealtimeConfig(drop_params=drop_params)
@ -396,7 +415,7 @@ class BedrockRealtime(BaseAWSLLM):
verbose_proxy_logger.debug("Bedrock Realtime: Bidirectional stream established")
if pending_session_update is None:
if pending_opening_message is None:
await websocket.send_text(json.dumps(transformation_config.session_created_event(model, logging_obj)))
verbose_proxy_logger.debug("Bedrock Realtime: sent session.created to client on connect")
@ -418,7 +437,7 @@ class BedrockRealtime(BaseAWSLLM):
model,
session_state,
logging_obj,
initial_message=pending_session_update,
initial_message=pending_opening_message,
)
logged_events: Final = (
@ -531,18 +550,22 @@ class BedrockRealtime(BaseAWSLLM):
if not await _reject_session_update(client_ws, parsed_client_message, rejection):
break
continue
client_ws.scope[BEDROCK_REALTIME_PENDING_SESSION_UPDATE_SCOPE_KEY] = message
transformed_messages = transformation_config.transform_realtime_request(
message=message,
model=model,
session_configuration_request=session_state.get("session_configuration_request"),
)
for bedrock_message in transformed_messages:
await send_to_bedrock(bedrock_message)
if transformed_messages and not _is_session_committed(client_ws.scope):
await _send_opening_message(
client_ws, bedrock_stream, message, transformed_messages, send_to_bedrock
)
else:
for bedrock_message in transformed_messages:
await send_to_bedrock(bedrock_message)
if is_session_update and not await _ack_session_update(
client_ws, bedrock_stream, transformation_config, model, logging_obj, parsed_client_message
client_ws, transformation_config, model, logging_obj, parsed_client_message
):
break
finally:

123
litellm/llms/bedrock/realtime/transformation.py
View file

@ -123,26 +123,81 @@ def _integer_setting(value: JsonValue) -> int | None:
return None if number is None else int(number)
def _malformed_number_settings(session_config: Mapping[str, JsonValue]) -> tuple[str, ...]:
def requested_modalities(session_config: Mapping[str, JsonValue]) -> tuple[str, ...] | None:
modalities: Final = session_config.get("modalities")
if not isinstance(modalities, list):
return None
names: Final = tuple(modality for modality in modalities if isinstance(modality, str))
return names if len(names) == len(modalities) else None
_OPENAI_VOICES: Final = frozenset(
("alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse", "marin", "cedar")
)
def _unplayable_voice(session_config: Mapping[str, JsonValue]) -> str | None:
voice: Final = session_config.get("voice")
if not isinstance(voice, str):
return None
return voice if not voice.strip() or voice.lower() in _OPENAI_VOICES else None
def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[str, str], ...]:
instructions: Final = session_config.get("instructions")
voice: Final = session_config.get("voice")
input_audio_format: Final = session_config.get("input_audio_format")
output_audio_format: Final = session_config.get("output_audio_format")
modalities: Final = session_config.get("modalities")
temperature: Final = session_config.get("temperature")
max_output_tokens: Final = session_config.get("max_response_output_tokens")
input_rate_hertz: Final = session_config.get("input_sample_rate_hertz")
output_rate_hertz: Final = session_config.get("output_sample_rate_hertz")
tools: Final = session_config.get("tools")
malformed: Final = (
("temperature", temperature is not None and _number_setting(temperature) is None),
("instructions", "a string", instructions is not None and not isinstance(instructions, str)),
("voice", "a string", voice is not None and not isinstance(voice, str)),
("tools", "a list", tools is not None and not isinstance(tools, list)),
(
"input_audio_format",
"a string",
input_audio_format is not None and not isinstance(input_audio_format, str),
),
(
"output_audio_format",
"a string",
output_audio_format is not None and not isinstance(output_audio_format, str),
),
(
"modalities",
"a list of strings",
modalities is not None and requested_modalities(session_config) is None,
),
("temperature", "a number", temperature is not None and _number_setting(temperature) is None),
(
"max_response_output_tokens",
"a number or 'inf'",
max_output_tokens is not None
and max_output_tokens != "inf"
and _integer_setting(max_output_tokens) is None,
),
("output_sample_rate_hertz", output_rate_hertz is not None and _integer_setting(output_rate_hertz) is None),
(
"input_sample_rate_hertz",
"a number",
input_rate_hertz is not None and _integer_setting(input_rate_hertz) is None,
),
(
"output_sample_rate_hertz",
"a number",
output_rate_hertz is not None and _integer_setting(output_rate_hertz) is None,
),
)
return tuple(name for name, is_malformed in malformed if is_malformed)
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
@dataclasses.dataclass(frozen=True, slots=True)
class BedrockSessionUpdateRejection:
code: Literal["invalid_type", "session_update_not_applied"]
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
message: str
param: str
@ -312,7 +367,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
return 8000 # G.711 typically uses 8kHz
return 24000 if is_output else 16000
def transform_session_update_event(self, json_message: dict) -> list[str]:
def transform_session_update_event(self, json_message: Mapping[str, JsonValue]) -> list[str]:
"""
Transform session.update event to Bedrock session configuration.
@ -325,7 +380,10 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
verbose_logger.debug("Handling session.update")
messages: Final[list[str]] = []
session_config: Final = json_message.get("session", {})
requested_session: Final = json_message.get("session")
session_config: Final[Mapping[str, JsonValue]] = (
requested_session if isinstance(requested_session, dict) else {}
)
locked_settings: Final = self._apply_locked_settings(session_config, self._started_settings)
# Settings Bedrock locks at prompt start may only change before it
@ -336,13 +394,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self.output_sample_rate_hertz = locked_settings.output_sample_rate_hertz
# Update audio input configuration from session if provided
if "input_audio_format" in session_config:
input_format: Final = session_config["input_audio_format"]
input_format: Final = session_config.get("input_audio_format")
if isinstance(input_format, str):
self.input_sample_rate_hertz = self._map_audio_format_to_sample_rate(input_format, is_output=False)
# Allow direct override of the input sample rate if provided (custom extension)
if "input_sample_rate_hertz" in session_config:
self.input_sample_rate_hertz = session_config["input_sample_rate_hertz"]
requested_input_rate: Final = _integer_setting(session_config.get("input_sample_rate_hertz"))
if requested_input_rate is not None:
self.input_sample_rate_hertz = requested_input_rate
if self.prompt_started:
return messages
@ -352,7 +411,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
messages.append(json.dumps(session_start))
# Send prompt start
tools: Final[Sequence[JsonValue] | None] = session_config.get("tools")
tools: Final = session_config.get("tools")
prompt_start_config: Final[Mapping[str, object]] = {
"promptName": self.prompt_name,
"textOutputConfiguration": {"mediaType": self.text_media_type},
@ -370,7 +429,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
"toolUseOutputConfiguration": {"mediaType": "application/json"},
"toolConfiguration": {"tools": self._transform_tools_to_bedrock_format(tools)},
}
if tools
if isinstance(tools, list) and tools
else {}
),
}
@ -445,12 +504,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
if isinstance(requested_tools, list)
else base.tools_json
)
system_prompt: Final = base.system_prompt if instructions is None else str(instructions)
voice_id: Final = base.voice_id if voice is None else str(voice)
system_prompt: Final = instructions if isinstance(instructions, str) else base.system_prompt
voice_id: Final = (
voice if isinstance(voice, str) and _unplayable_voice(session_config) is None else base.voice_id
)
format_sample_rate: Final = (
base.output_sample_rate_hertz
if output_audio_format is None
else self._map_audio_format_to_sample_rate(str(output_audio_format), is_output=True)
self._map_audio_format_to_sample_rate(output_audio_format, is_output=True)
if isinstance(output_audio_format, str)
else base.output_sample_rate_hertz
)
requested_output_rate: Final = _integer_setting(session_config.get("output_sample_rate_hertz"))
output_sample_rate_hertz: Final = (
@ -476,16 +537,26 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
session_config: Final = json_message.get("session")
if not isinstance(session_config, Mapping):
return None
malformed: Final = () if self.drop_params else _malformed_number_settings(session_config)
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
if malformed:
name, expected = malformed[0]
return BedrockSessionUpdateRejection(
code="invalid_type",
message=(
f"Invalid type for 'session.{malformed[0]}': Bedrock Nova Sonic needs a number for temperature "
"and output_sample_rate_hertz, and a number or 'inf' for max_response_output_tokens. "
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
"Set drop_params to drop it instead"
),
param=f"session.{malformed[0]}",
param=f"session.{name}",
)
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
if unplayable_voice is not None:
return BedrockSessionUpdateRejection(
code="invalid_value",
message=(
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
"Set drop_params to drop it instead"
),
param="session.voice",
)
if not self.prompt_started:
return None
@ -785,7 +856,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
List of Bedrock format messages (JSON strings)
"""
try:
json_message: Final[dict[str, object]] = json.loads(message)
json_message: Final[dict[str, JsonValue]] = json.loads(message)
except json.JSONDecodeError:
verbose_logger.warning("Invalid JSON message: %s", message[:200])
return []
@ -816,11 +887,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self,
model: str,
logging_obj: LiteLLMLoggingObj,
modalities: list[str] | None = None,
modalities: Sequence[str] | None = None,
) -> OpenAIRealtimeStreamSession:
session: Final = OpenAIRealtimeStreamSession(
id=logging_obj.litellm_trace_id,
modalities=modalities if modalities is not None else ["text", "audio"],
modalities=list(modalities) if modalities is not None else ["text", "audio"],
)
if model is not None and isinstance(model, str):
session["model"] = model
@ -842,7 +913,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
self,
model: str,
logging_obj: LiteLLMLoggingObj,
modalities: list[str] | None = None,
modalities: Sequence[str] | None = None,
) -> OpenAIRealtimeStreamSessionEvents:
"""Build the OpenAI session.updated ack reflecting the client's requested modalities."""
return OpenAIRealtimeStreamSessionEvents(

4
litellm/llms/custom_httpx/container_handler.py
View file

@ -5,6 +5,7 @@ This module provides a single generic handler that can process any container fil
endpoint defined in endpoints.json, eliminating the need for individual handler methods.
"""
import asyncio
import json
from collections.abc import Coroutine, Mapping, Sequence
from pathlib import Path
@ -236,6 +237,7 @@ def _request_headers(
return container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
@ -435,7 +437,7 @@ class GenericContainerHandler:
http_client: Final = _async_http_client(client, litellm_params)
# Build request
headers = _request_headers(container_provider_config, extra_headers, litellm_params)
headers = await asyncio.to_thread(_request_headers, container_provider_config, extra_headers, litellm_params)
if extra_headers:
headers.update(extra_headers)

204
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -1515,8 +1515,65 @@ class BaseLLMHTTPHandler:
api_base: str,
api_key: str | None,
headers: Mapping[str, str],
) -> tuple[str, dict[str, str], dict[str, object]]:
outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key)
litellm_params: Mapping[str, object],
) -> tuple[str, Mapping[str, str], Mapping[str, object]]:
outbound_headers: Final = provider_config.validate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
return self._prepare_decisions_request_with_headers(
model=model,
logging_obj=logging_obj,
provider_config=provider_config,
body=body,
logged_body=logged_body,
api_base=api_base,
api_key=api_key,
outbound_headers=outbound_headers,
)
async def _aprepare_decisions_request(
self,
model: str,
logging_obj: LiteLLMLoggingObj | None,
provider_config: BaseDecisionsConfig,
body: Mapping[str, object],
logged_body: Mapping[str, object],
api_base: str,
api_key: str | None,
headers: Mapping[str, str],
litellm_params: Mapping[str, object],
) -> tuple[str, Mapping[str, str], Mapping[str, object]]:
outbound_headers: Final = await provider_config.avalidate_environment(
headers=headers,
model=model,
api_key=api_key,
litellm_params=litellm_params,
)
return self._prepare_decisions_request_with_headers(
model=model,
logging_obj=logging_obj,
provider_config=provider_config,
body=body,
logged_body=logged_body,
api_base=api_base,
api_key=api_key,
outbound_headers=outbound_headers,
)
def _prepare_decisions_request_with_headers(
self,
model: str,
logging_obj: LiteLLMLoggingObj | None,
provider_config: BaseDecisionsConfig,
body: Mapping[str, object],
logged_body: Mapping[str, object],
api_base: str,
api_key: str | None,
outbound_headers: Mapping[str, str],
) -> tuple[str, Mapping[str, str], Mapping[str, object]]:
url: Final = provider_config.get_complete_url(api_base=api_base, model=model)
data: Final = dict(body)
if logging_obj is not None:
@ -1553,6 +1610,7 @@ class BaseLLMHTTPHandler:
api_base=api_base,
api_key=api_key,
headers=headers,
litellm_params=litellm_params,
)
signed_headers, signed_body = provider_config.sign_request(
headers=outbound_headers,
@ -1567,7 +1625,7 @@ class BaseLLMHTTPHandler:
response: Final = sync_httpx_client.post(
url,
content=signed_body,
json=data if signed_body is None else None,
json=dict(data) if signed_body is None else None,
headers=dict(signed_headers),
timeout=timeout,
logging_obj=logging_obj,
@ -1594,7 +1652,7 @@ class BaseLLMHTTPHandler:
litellm_params: Mapping[str, object],
client: AsyncHTTPHandler | None = None,
) -> DecisionsIRResponse:
url, outbound_headers, data = self._prepare_decisions_request(
url, outbound_headers, data = await self._aprepare_decisions_request(
model=model,
logging_obj=logging_obj,
provider_config=provider_config,
@ -1603,6 +1661,7 @@ class BaseLLMHTTPHandler:
api_base=api_base,
api_key=api_key,
headers=headers,
litellm_params=litellm_params,
)
sign: Final = partial(
provider_config.sign_request,
@ -1625,7 +1684,7 @@ class BaseLLMHTTPHandler:
response: Final = await async_httpx_client.post(
url,
content=signed_body,
json=data if signed_body is None else None,
json=dict(data) if signed_body is None else None,
headers=dict(signed_headers),
timeout=timeout,
logging_obj=logging_obj,
@ -7127,7 +7186,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = image_edit_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
image_edit_provider_config.validate_environment,
api_key=litellm_params.api_key,
headers=image_edit_optional_request_params.get("extra_headers", {}) or {},
model=model,
@ -7586,7 +7646,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_generation_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_generation_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=video_generation_optional_request_params.get("extra_headers", {}) or {},
model=model,
@ -7781,7 +7842,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_content_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_content_provider_config.validate_environment,
headers=extra_headers or {},
model="",
api_key=api_key,
@ -7961,7 +8023,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_remix_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_remix_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8124,7 +8187,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8264,7 +8328,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8436,7 +8501,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8622,7 +8688,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_provider_config.validate_environment,
api_key=api_key or litellm_params.get("api_key", None),
headers=extra_headers or {},
model="",
@ -8754,7 +8821,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_list_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_list_provider_config.validate_environment,
api_key=api_key,
headers=extra_headers or {},
model="",
@ -8835,7 +8903,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_delete_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_delete_provider_config.validate_environment,
api_key=api_key,
headers=extra_headers or {},
model="",
@ -9019,7 +9088,8 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = video_status_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
video_status_provider_config.validate_environment,
api_key=api_key,
headers=extra_headers or {},
model="",
@ -9122,6 +9192,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
# Add Content-Type header for JSON requests
@ -9197,9 +9268,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
# Add Content-Type header for JSON requests
@ -9293,6 +9366,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9369,9 +9443,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9460,6 +9536,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9536,9 +9613,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9629,6 +9708,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9705,9 +9785,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9803,6 +9885,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9881,9 +9964,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -9972,6 +10057,7 @@ class BaseLLMHTTPHandler:
headers: Final = container_provider_config.validate_environment(
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -10047,9 +10133,11 @@ class BaseLLMHTTPHandler:
async_httpx_client = client
# Validate environment and get headers
headers: Final = container_provider_config.validate_environment(
headers: Final = await asyncio.to_thread(
container_provider_config.validate_environment,
headers=extra_headers or {},
api_key=litellm_params.get("api_key", None),
litellm_params=litellm_params,
)
if extra_headers:
@ -10167,8 +10255,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10410,8 +10500,10 @@ class BaseLLMHTTPHandler:
vector_store_provider_config.validate_create_vector_store()
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10541,8 +10633,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10663,8 +10757,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10805,8 +10901,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -10950,8 +11048,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -11068,8 +11168,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
@ -11210,8 +11312,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11351,8 +11455,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11482,8 +11588,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11619,8 +11727,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
@ -11762,8 +11872,10 @@ class BaseLLMHTTPHandler:
else:
async_httpx_client = client
headers: Final = vector_store_files_provider_config.validate_environment(
headers=extra_headers or {}, litellm_params=litellm_params
headers: Final = await asyncio.to_thread(
vector_store_files_provider_config.validate_environment,
headers=extra_headers or {},
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)

102
litellm/llms/dashscope/videos/transformation.py
View file

@ -3,13 +3,10 @@ DashScope (Alibaba Model Studio) async video task API: create, poll and download
delete or remix endpoint, so those raise.
"""
import base64
import json
from collections.abc import Mapping
from datetime import datetime
from io import BufferedReader, BytesIO
from math import log
from pathlib import Path, PurePath
from types import MappingProxyType
from typing import TYPE_CHECKING, Annotated, Final
@ -19,9 +16,13 @@ from pydantic import BaseModel, Field, StrictBool, StrictInt, StrictStr, TypeAda
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.images.utils import ImageEditRequestUtils
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.videos.async_task_utils import (
input_reference_url,
polled_video_models,
routed_video_models,
)
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # house cached-client factory has no public alias and its stub leaves params untyped
@ -32,7 +33,6 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ModelInfo
from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject
from litellm.types.videos.utils import (
decode_video_id_with_provider,
encode_video_id_with_provider,
extract_original_video_id,
)
@ -86,7 +86,6 @@ _EMPTY_PARAMS: Final[dict[str, object]] = {} # mutable-ok: BaseVideoConfig cont
_TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_TaskResponse)
_PARAMETERS_ADAPTER: Final = TypeAdapter(dict[str, object])
_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
_STATUS_MAP: Final = MappingProxyType(
{
@ -383,39 +382,6 @@ def _is_priced_tier(resolution: object, priced_tiers: tuple[str, ...]) -> bool:
return isinstance(resolution, str) and resolution.strip().upper() in priced_tiers
def _read_all_bytes(file_obj: object) -> bytes:
if isinstance(file_obj, tuple):
file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
return _read_all_bytes(file_tuple[1] if len(file_tuple) >= 2 else None)
if isinstance(file_obj, PurePath):
return Path(file_obj).read_bytes()
if isinstance(file_obj, (BytesIO, BufferedReader)):
current_position: Final = file_obj.tell()
file_obj.seek(0)
content: Final = file_obj.read()
file_obj.seek(current_position)
return content
if isinstance(file_obj, bytes):
return file_obj
if isinstance(file_obj, bytearray):
return bytes(file_obj)
read: Final = getattr(file_obj, "read", None)
if callable(read):
data: Final = read()
if isinstance(data, bytes):
return data
raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
def _image_url(image: object) -> str:
if isinstance(image, str):
return image
content: Final = _read_all_bytes(image)
content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
encoded: Final = base64.b64encode(content).decode("utf-8")
return f"data:{content_type};base64,{encoded}"
def _media_reference_type(model: str) -> str | None:
"""Models without the field predate the media array and take the image on the flat ``img_url`` field."""
info: Final = _dashscope_model_info(model)
@ -558,58 +524,6 @@ def _video_url_from_task(task: _TaskResponse, raw_response: httpx.Response) -> s
raise _video_error(raw_response, 502, "Video URL not found in task response. The task may not have succeeded yet.")
def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
litellm_params: Final[object] = getattr(logging_obj, "litellm_params", None)
try:
return _PARAMETERS_ADAPTER.validate_python(litellm_params)
except ValidationError:
return _EMPTY_PARAMS
def _polled_models(logging_obj: object) -> tuple[str | None, str | None]:
"""
The proxy routes by the models encoded in the polled id, so they must survive into the returned id. The served
group survives only when the router sent this poll to it, so an edited id cannot write any name it likes back.
"""
video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
if not isinstance(video_id, str):
return None, None
decoded: Final = decode_video_id_with_provider(video_id)
served: Final = decoded.get("served_model")
routed: Final = _poll_models(logging_obj)
return decoded.get("model_id") or None, served if routed is None or served in routed else None
def _poll_models(logging_obj: object) -> tuple[str, str | None] | None:
"""
A later status or content request checks the key's model access against the model encoded in the id, and
routes by it. DashScope serves the same model name from several regional hosts, so only the group that
served the task can send its polls back to the region that created it, while a key may be scoped to the
alias, routing group or wildcard name it asked for instead. The id carries both names: the one the router
was asked for on this hop, and the group that served it when that differs and is not a wildcard. The router
stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
"""
params: Final = _logged_litellm_params(logging_obj)
stamped: Final = (_stamped_poll_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
return next((models for models in stamped if models), None)
def _stamped_poll_models(metadata: object) -> tuple[str, str | None] | None:
try:
fields: Final = _PARAMETERS_ADAPTER.validate_python(metadata)
except ValidationError:
return None
requested: Final = _id_safe_name(fields.get("model_group"))
served: Final = _id_safe_name(fields.get("deployment_model_name"))
if requested is None:
return None
return requested, None if served is None or served == requested or "*" in served else served
def _id_safe_name(name: object) -> str | None:
return name if isinstance(name, str) and name and ";" not in name else None
class DashScopeVideoConfig(BaseVideoConfig):
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseVideoConfig contract returns list
return [
@ -804,7 +718,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
input_reference: Final = video_create_optional_request_params.get("input_reference")
if input_reference is None:
return None, None
image_url: Final = _image_url(input_reference)
image_url: Final = input_reference_url(input_reference)
media_type: Final = _media_reference_type(model)
if media_type is None:
return "img_url", image_url
@ -823,7 +737,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
requested: Final = self._requested_parameters(request_data)
video_obj: Final = _video_object_from_task(task, raw_response, model=model, requested=requested)
if custom_llm_provider and video_obj.id:
requested_model, served_model = _poll_models(logging_obj) or (model, None)
requested_model, served_model = routed_video_models(logging_obj) or (model, None)
video_obj.id = encode_video_id_with_provider(
video_obj.id, custom_llm_provider, requested_model, served_model
)
@ -869,7 +783,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
client: "HTTPHandler | None" = None,
) -> VideoObject:
task: Final = _parse_task_response(raw_response)
model, served_model = _polled_models(logging_obj)
model, served_model = polled_video_models(logging_obj)
video_obj: Final = _video_object_from_task(task, raw_response, model=model)
if custom_llm_provider and video_obj.id:
video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model, served_model)

4
litellm/llms/databricks/chat/transformation.py
View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
merge_consecutive_system_messages,
strip_litellm_internal_message_fields,
strip_name_from_message,
strip_prefill_prefix_field,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.base_utils import (
@ -487,6 +488,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
Databricks does not support:
- 'name' in user message.
- litellm's internal `thinking_blocks` / `reasoning_content` on assistant messages.
- the `prefix` prefill flag Mistral and DeepSeek read, which Claude does not need to continue an assistant turn.
- non-Claude json_object "json" checks read only string content, so a single text block is sent as a string.
"""
new_messages = []
@ -496,7 +498,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
else:
_message = message
_message = strip_name_from_message(_message, allowed_name_roles=["user"])
_message = strip_litellm_internal_message_fields(_message)
_message = strip_prefill_prefix_field(strip_litellm_internal_message_fields(_message))
# Move message-level cache_control into a content block when content is a string.
if "cache_control" in _message and isinstance(_message.get("content"), str):
_message = self._move_cache_control_into_string_content_block(_message)

13
litellm/llms/gemini/common_utils.py
View file

@ -14,7 +14,7 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
GEMINI_IMAGE_ASPECT_RATIOS: Final[dict[str, float]] = {
"1:1": 1 / 1,
@ -497,12 +497,14 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
system: object | None = None,
messages_format: TokenCountMessagesFormat = "openai",
output_format: JsonValue = None,
context_management: CountContextManagement | None = None,
) -> TokenCountResponse | None:
import copy
from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
from litellm.llms.vertex_ai.count_tokens.context import (
VertexGeminiCountContextError,
count_context_edits,
count_context_error_response,
gemini_count_contents,
inference_request_view,
@ -511,7 +513,14 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
vertex_gemini_count_context,
)
request_view: Final = inference_request_view(messages or [], system, messages_format)
request_view: Final = await inference_request_view(
messages or [],
system,
messages_format,
count_context_edits(context_management, tools, (deployment or {}).get("litellm_params", {}), model_to_use),
)
if isinstance(request_view, VertexGeminiCountContextError):
return count_context_error_response(request_view, request_model, model_to_use)
count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
if isinstance(count_context, VertexGeminiCountContextError):
return count_context_error_response(count_context, request_model, model_to_use)

11
litellm/llms/gemini/realtime/transformation.py
View file

@ -161,6 +161,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"""Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it."""
return True
def _rejects_vad_tuning_with_detection_off(self) -> bool:
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
return True
@staticmethod
def _usage_detail_alias(details: Mapping[str, int | None] | None, defaults: dict[str, int]) -> dict[str, int]:
if not isinstance(details, dict):
@ -279,10 +283,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
OpenAI ``semantic_vad`` has no Gemini Live equivalent — return an empty
dict so callers omit ``realtimeInputConfig`` (mapping it with
``disabled: true`` breaks native-audio sessions).
``disabled: true`` breaks native-audio sessions). An explicit
``create_response: false`` maps to detection off alone where the backend
rejects tuning next to ``disabled: true``.
"""
if isinstance(value, dict) and value.get("type") == "semantic_vad" and "create_response" not in value:
return AutomaticActivityDetection()
auto_response_off: Final = "create_response" in value and value["create_response"] is False
if auto_response_off and self._rejects_vad_tuning_with_detection_off():
return AutomaticActivityDetection(disabled=True)
automatic_activity_dection: Final = AutomaticActivityDetection()
if "create_response" in value and isinstance(value["create_response"], bool):

17
litellm/llms/meta/realtime/transformation.py
View file

@ -26,6 +26,7 @@ from litellm.llms.base_llm.realtime.transcription_protocol import (
speech_event,
transcription_session,
transcription_session_created_event,
uses_server_vad,
)
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.secret_managers.main import get_secret_str
@ -230,15 +231,7 @@ def _parse_sample_rate(update: TranscriptionSessionUpdate) -> MuseSampleRate:
return 16_000 if rate == 16_000 else 24_000
def _parse_mode(update: TranscriptionSessionUpdate) -> MuseMode:
if update.turn_detection_disabled:
return "PUSH_TO_TALK"
if update.turn_detection_type not in (None, "server_vad"):
raise MuseProtocolError("Muse Voice supports server_vad turn detection or null")
return "ENDPOINTING"
def parse_session_update(payload: str, expected_model: str) -> MuseSessionConfig:
def parse_session_update(payload: str, expected_model: str, *, drop_params: bool = False) -> MuseSessionConfig:
update: Final = parse_transcription_session_update(payload, MuseProtocolError)
if update.session_type not in (None, "transcription", "realtime"):
raise MuseProtocolError("Muse Voice supports transcription sessions only")
@ -253,7 +246,7 @@ def parse_session_update(payload: str, expected_model: str) -> MuseSessionConfig
raise MuseProtocolError("realtime session model cannot be changed")
return MuseSessionConfig(
model=normalized_model,
mode=_parse_mode(update),
mode="ENDPOINTING" if uses_server_vad(update, "Muse Voice", drop_params, MuseProtocolError) else "PUSH_TO_TALK",
sample_rate=_parse_sample_rate(update),
language_bias=() if update.language is None else (normalize_language(update.language),),
)
@ -441,9 +434,11 @@ class MetaRealtimeConfig(BaseRealtimeConfig):
*,
monotonic: Callable[[], float] = time.monotonic,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
drop_params: bool = False,
) -> None:
self._monotonic: Final = monotonic
self._sleep: Final = sleep
self._drop_params: Final = drop_params
self._transformer: Final = MuseEventTransformer()
self._access_token: str | None = None
self._config: MuseSessionConfig | None = None
@ -553,7 +548,7 @@ class MetaRealtimeConfig(BaseRealtimeConfig):
access_token: Final = self._access_token
if access_token is None:
raise MuseProtocolError("Meta API key was not validated before the session was configured")
config: Final = parse_session_update(message, model)
config: Final = parse_session_update(message, model, drop_params=self._drop_params)
self._config = config
self._transformer.configure(config)
return (json.dumps(config.handshake(access_token), separators=(",", ":")),)

9
litellm/llms/nace/decisions/transformation.py Normal file
View file

@ -0,0 +1,9 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class NaceDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("NACE_API_KEY",)
api_base_env = ("NACE_API_BASE", "NACE_BASE_URL")
def get_default_api_base(self) -> str | None:
return "https://console.nace.ai"

11
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -39,6 +39,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
list_field,
merge_guardrailed_scoped_messages,
merge_returned_tools_into_request_tools,
openai_tool_name,
@ -156,7 +157,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs["structured_messages"] = [structured_messages[index] for index in scoped_message_indices]
# Pass tools (function definitions) to the guardrail
tools: Final = data.get("tools")
if tools and not scan_only_tool_results:
if isinstance(tools, list) and tools and not scan_only_tool_results:
inputs["tools"] = tools
# Include model information if available
model: Final = data.get("model")
@ -258,19 +259,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
return "no scannable content after message scoping"
return None if images else "no scannable content"
def extract_request_tool_names(self, data: dict) -> list[str]:
def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]:
"""Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name,
and the tools a replayed tool_search_output loads)."""
names: Final[list[str]] = []
for tool in data.get("tools") or []:
for tool in list_field(data, "tools"):
if isinstance(tool, dict) and tool.get("type") == "function":
fn = tool.get("function")
if isinstance(fn, dict) and fn.get("name"):
names.append(str(fn["name"]))
for fn in data.get("functions") or []:
for fn in list_field(data, "functions"):
if isinstance(fn, dict) and fn.get("name"):
names.append(str(fn["name"]))
return [*names, *replayed_tool_search_tool_names(cast(object, data))] # cast-ok: upcast, read field by field
return [*names, *replayed_tool_search_tool_names(data)]
def _extract_inputs(
self,

4
litellm/llms/openai/common_utils.py
View file

@ -407,6 +407,10 @@ class BaseOpenAILLM:
http2=http2_enabled(),
)
@classmethod
def get_sync_http_client(cls) -> httpx.Client | None:
return cls._get_sync_http_client()
class OpenAICredentials(NamedTuple):
api_base: str

71
litellm/llms/openai/completion/handler.py
View file

@ -1,9 +1,7 @@
import json
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import Final
from openai import AsyncOpenAI, OpenAI
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@ -12,7 +10,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
from litellm.utils import ProviderConfigManager
from ..common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIError, OpenAIHTTPClient
from ..common_utils import BaseOpenAILLM, OpenAIError
from ..workload_identity import build_async_openai_client, build_openai_client
from .transformation import OpenAITextCompletionConfig
@ -43,7 +42,7 @@ class OpenAITextCompletion(BaseLLM):
print_verbose: Callable | None = None,
api_base: str | None = None,
acompletion: bool = False,
litellm_params=None,
litellm_params: Mapping[str, object] | None = None,
logger_fn=None,
client=None,
organization: str | None = None,
@ -95,6 +94,7 @@ class OpenAITextCompletion(BaseLLM):
max_retries=max_retries,
client=client,
organization=organization,
litellm_params=litellm_params,
)
else:
return self.acompletion(
@ -109,6 +109,7 @@ class OpenAITextCompletion(BaseLLM):
max_retries=max_retries,
organization=organization,
client=client,
litellm_params=litellm_params,
)
elif optional_params.get("stream", False):
return self.streaming(
@ -123,19 +124,22 @@ class OpenAITextCompletion(BaseLLM):
max_retries=max_retries,
client=client,
organization=organization,
litellm_params=litellm_params,
)
else:
if client is None:
openai_client = OpenAI(
openai_client: Final = (
client
if client is not None
else build_openai_client(
api_key=api_key,
base_url=api_base,
http_client=litellm.client_session or OpenAIHTTPClient(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.client_session,
)
else:
openai_client = client
)
raw_response: Final = openai_client.completions.with_raw_response.create(**data)
response: Final = raw_response.parse()
@ -175,19 +179,22 @@ class OpenAITextCompletion(BaseLLM):
max_retries: int,
organization: str | None = None,
client=None,
litellm_params: Mapping[str, object] | None = None,
):
try:
if client is None:
openai_aclient = AsyncOpenAI(
openai_aclient: Final = (
client
if client is not None
else build_async_openai_client(
api_key=api_key,
base_url=api_base,
http_client=BaseOpenAILLM.get_async_http_client(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=BaseOpenAILLM.get_async_http_client,
)
else:
openai_aclient = client
)
raw_response: Final = await openai_aclient.completions.with_raw_response.create(**data)
response: Final = raw_response.parse()
@ -228,18 +235,21 @@ class OpenAITextCompletion(BaseLLM):
max_retries=None,
client=None,
organization=None,
litellm_params: Mapping[str, object] | None = None,
):
if client is None:
openai_client = OpenAI(
openai_client: Final = (
client
if client is not None
else build_openai_client(
api_key=api_key,
base_url=api_base,
http_client=litellm.client_session or OpenAIHTTPClient(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.client_session,
)
else:
openai_client = client
)
try:
raw_response: Final = openai_client.completions.with_raw_response.create(**data)
@ -285,18 +295,21 @@ class OpenAITextCompletion(BaseLLM):
api_base: str | None = None,
client=None,
organization=None,
litellm_params: Mapping[str, object] | None = None,
):
if client is None:
openai_client = AsyncOpenAI(
openai_client: Final = (
client
if client is not None
else build_async_openai_client(
api_key=api_key,
base_url=api_base,
http_client=litellm.aclient_session or OpenAIAsyncHTTPClient(),
api_base=api_base,
timeout=timeout,
max_retries=max_retries,
organization=organization,
litellm_params=litellm_params,
static_key_http_client_factory=lambda: litellm.aclient_session,
)
else:
openai_client = client
)
raw_response: Final = await openai_client.completions.with_raw_response.create(**data)
response: Final = raw_response.parse()

Some files were not shown because too many files have changed in this diff Show more